...
 
Commits (4)
......@@ -23,11 +23,11 @@ def human_size(bytes, units=[' bytes','KB','MB','GB','TB', 'PB', 'EB']):
def fetch_project(student):
path = 'academany/fabacademy{}'.format(student['path'])
print('path: {}'.format(path))
#print('path: {}'.format(path))
encoded = urllib.parse.quote_plus(path)
print('encoded: {}'.format(encoded))
#print('encoded: {}'.format(encoded))
url_notoken = 'https://gitlab.fabcloud.org/api/v4/projects/{}?statistics=true'.format(encoded)
print('url_notoken: {}'.format(url_notoken))
#print('url_notoken: {}'.format(url_notoken))
url_full = '{}&private_token={}'.format(url_notoken, TOKEN)
try:
response = requests.get(url_full)
......@@ -35,9 +35,9 @@ def fetch_project(student):
print('Requests response failed')
data = response.json()
print('Debug request')
print(response.text)
print('---')
#print('Debug request')
#print(response.text)
#print('---')
return data
def load_students():
......@@ -47,56 +47,56 @@ def load_students():
return students
def get_stats(student):
try:
project = fetch_project(student)
# print "%s\t%s\t%s\t%s" % ( student['name'], student['lab'], human_size(project['statistics']['storage_size']), human_size(project['statistics']['repository_size']) )
return {
"name": student['name'],
"username": student['username'],
"lab": student['lab'],
"path": student['path'],
"storage": human_size(project['statistics']['storage_size']),
"storage_size": project['statistics']['storage_size'],
"repo": human_size(project['statistics']['repository_size']),
"repo_size": project['statistics']['repository_size'],
"continuing": student['code'][:4] != 'FA19'
}
except:
# traceback.print_exc()
print('WARNING stats JSON for student ' + student['name'] + 'are not available')
pass
try:
project = fetch_project(student)
# print "%s\t%s\t%s\t%s" % ( student['name'], student['lab'], human_size(project['statistics']['storage_size']), human_size(project['statistics']['repository_size']) )
return {
"name": student['name'],
"username": student['username'],
"lab": student['lab'],
"path": student['path'],
"storage": human_size(project['statistics']['storage_size']),
"storage_size": project['statistics']['storage_size'],
"repo": human_size(project['statistics']['repository_size']),
"repo_size": project['statistics']['repository_size'],
"continuing": student['code'][:4] != 'FA19'
}
except:
# traceback.print_exc()
print('WARNING stats JSON for student ' + student['name'] + 'are not available')
pass
if __name__ == '__main__':
print('File sizes utility - v0.1')
print('Using %d threads' % NUM_THREAD)
pool = Pool(NUM_THREAD)
print('Loading students')
start = time.time()
students = load_students()
end = time.time()
print('Done. %d students loaded in %ds' % (len(students), end-start))
print('Loading stats')
start = time.time()
stats = pool.map(get_stats, students)
end = time.time()
print('Done. Stats loaded in %ds' % (end - start))
print('\n')
print('File sizes utility - v0.1')
print('Using %d threads' % NUM_THREAD)
pool = Pool(NUM_THREAD)
print('Loading students')
start = time.time()
students = load_students()
end = time.time()
print('Done. %d students loaded in %ds' % (len(students), end-start))
print('Loading stats')
start = time.time()
stats = pool.map(get_stats, students)
end = time.time()
print('Done. Stats loaded in %ds' % (end - start))
print('\n')
output = []
output = []
output.append('### TOP 10 - Total Repository size - all branches')
output.append('------------------')
output.append('Also see [Gitlab pages size](./fs.md)')
output.append('')
by_repo = sorted(stats, key=lambda x: int(x["repo_size"]))
by_repo.reverse()
output.append('pos|username|size|continuing')
output.append('---|--------|----|----------')
idx=1
for s in by_repo[:10]:
output.append( "%d.|[%s](https://fabacademy.org%s)|%s|%s" % (idx, s["username"],s["path"], s["repo"], s["continuing"] and "yes" or ""))
idx = idx+1
output.append( '\n' )
output.append('### TOP 10 - Total Repository size - all branches')
output.append('------------------')
output.append('Also see [Gitlab pages size](./fs.md)')
output.append('')
by_repo = sorted(stats, key=lambda student: int(student.repo_size))
by_repo.reverse()
output.append('pos|username|size|continuing')
output.append('---|--------|----|----------')
idx=1
for s in by_repo[:10]:
output.append( "%d.|[%s](https://fabacademy.org%s)|%s|%s" % (idx, s["username"],s["path"], s["repo"], s["continuing"] and "yes" or ""))
idx = idx+1
output.append( '\n' )
# output.append( '### Top 10 - Storage size' )
# output.append( '---------------------' )
......@@ -109,7 +109,7 @@ if __name__ == '__main__':
# output.append( "%d.|[%s](https://fabacademy.org%s)|%s|%s" % (idx, s["username"],s["path"], s["storage"], s["continuing"] and "yes" or ""))
# idx = idx + 1
with open('sizes.md', 'w') as outfile:
outfile.write('\n'.join(output))
print('Wrote sizes.md')
sys.exit(0)
with open('sizes.md', 'w') as outfile:
outfile.write('\n'.join(output))
print('Wrote sizes.md')
sys.exit(0)