crapaud-fou/crazy-toads.github.io
2
0
Fork 0
Code Issues Pull requests Releases Activity

refacto

Browse source
This commit is contained in:
Mickael 2019-07-04 21:15:35 +02:00
parent 2c188c4ccf
commit f7baf818e1
3 changed files with 133 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
script/common/channelhelper.py
View file

@ -36,4 +36,19 @@ class Tsunami:
PROJECT = 1 << 1 PROJECT = 1 << 1
DEMOCRACY = 1 << 2 DEMOCRACY = 1 << 2
ECOLOGY = 1 << 3 ECOLOGY = 1 << 3
TECHNOLOGY = 1 << 4 TECHNOLOGY = 1 << 4
def getAllChannels(rocket):
index = 0
allChannels = []
while True:
channels = rocket.channels_list(offset= index).json()
allChannels.extend([ channel for channel in channels['channels'] if 'archived' not in channel])
if channels['count'] + channels['offset'] >= channels['total']:
break
index += channels['count']
return allChannels
if __name__ == "__main__":
print("Ce fichier est juste une librarie")

97
script/getchannels.py
View file

@ -9,40 +9,35 @@ import json
import dev_config as cfg import dev_config as cfg
import os import os
import re import re
from common.channelhelper import getNodesOrigin from common.channelhelper import getNodesOrigin, getAllChannels, Tsunami
colorInfo = { def main():
'global': 'orange',
'technologie': 'gray', colorInfo = {
'democratie': 'red', 'global': 'orange',
'ecologie': 'green', 'technologie': 'gray',
'project': 'blue' 'democratie': 'red',
} 'ecologie': 'green',
'project': 'blue'
}
rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], server_url='https://coa.crapaud-fou.org') rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], server_url=cfg.rocket['server'])
sizebase = 100 sizebase = 100
datas = [] datas = []
datas.append( { 'data':{'id':'global', 'label': 'global', 'size': sizebase, 'color': colorInfo['global'], 'href': 'https://coa.crapaud-fou.org/'}}) datas.append( { 'data':{'id':'global', 'label': 'global', 'size': sizebase, 'color': colorInfo['global'], 'href': 'https://coa.crapaud-fou.org/'}})
datas.append( { 'data':{'id':'ecologie', 'label': 'ecologie', 'size': sizebase, 'color': colorInfo['ecologie'], 'href': 'https://coa.crapaud-fou.org/'}}) datas.append( { 'data':{'id':'ecologie', 'label': 'ecologie', 'size': sizebase, 'color': colorInfo['ecologie'], 'href': 'https://coa.crapaud-fou.org/'}})
datas.append( { 'data':{'id':'democratie', 'label': 'democratie', 'size': sizebase, 'color': colorInfo['democratie'], 'href': 'https://coa.crapaud-fou.org/'}}) datas.append( { 'data':{'id':'democratie', 'label': 'democratie', 'size': sizebase, 'color': colorInfo['democratie'], 'href': 'https://coa.crapaud-fou.org/'}})
datas.append( { 'data':{'id':'technologie', 'label': 'technologie', 'size': sizebase, 'color': colorInfo['technologie'], 'href': 'https://coa.crapaud-fou.org/'}}) datas.append( { 'data':{'id':'technologie', 'label': 'technologie', 'size': sizebase, 'color': colorInfo['technologie'], 'href': 'https://coa.crapaud-fou.org/'}})
datas.append( { 'data':{'id':'project', 'label': 'projet', 'size': sizebase, 'color': colorInfo['project'], 'href': 'https://coa.crapaud-fou.org/'}}) datas.append( { 'data':{'id':'project', 'label': 'projet', 'size': sizebase, 'color': colorInfo['project'], 'href': 'https://coa.crapaud-fou.org/'}})
cohortes = { 'fr': { 'updateMap': 'france_fr'}} cohortes = { 'fr': { 'updateMap': 'france_fr'}}
cohortescolor = { 'fr': 'green' } cohortescolor = { 'fr': 'green' }
index = 0 nbChannels = 0
nbChannels = 0 nbCohorte = 0
nbCohorte = 0 totalChannels = 0
totalChannels = 0 for channel in getAllChannels(rocket):
while True:
channels = rocket.channels_list(offset= index).json()
totalChannels = channels['total']
for channel in channels['channels']:
print("{}".format(channel['name'])) print("{}".format(channel['name']))
if ('archived' in channel) and channel['archived']:
continue
if channel['name'].find('cohorte') != -1: if channel['name'].find('cohorte') != -1:
if 'description' in channel: if 'description' in channel:
@ -64,7 +59,7 @@ while True:
'label': channel['fname'] if 'fname' in channel else channel['name'], 'label': channel['fname'] if 'fname' in channel else channel['name'],
'size': size, 'size': size,
'color': 'grey', 'color': 'grey',
'href': 'https://coa.crapaud-fou.org/channel/'+channel['name'] 'href': f"{cfg.rocket['server']}/channel/{channel['name']}"
} }
} }
datas.append(node) datas.append(node)
@ -76,31 +71,29 @@ while True:
nbChannels += 1 nbChannels += 1
if channels['count'] + channels['offset'] >= channels['total']: # Récupération du répertoire racine du repo
break rootFolder = os.path.join(os.path.dirname(__file__), '..')
index += channels['count'] # Répertoire pour stocker le fichier de sortie
dataFolder = os.path.join(rootFolder, 'public','data')
# Faut il essayer de le créer au cas ou?
# os.makedirs(dataFolderPath, exist_ok=True)
channelsFilePath = os.path.abspath(os.path.join(dataFolder,'channelslist.json'))
# Récupération du répertoire racine du repo #print("Ecriture dans : "+channelsFilePath)
rootFolder = os.path.join(os.path.dirname(__file__), '..')
# Répertoire pour stocker le fichier de sortie
dataFolder = os.path.join(rootFolder, 'public','data')
# Faut il essayer de le créer au cas ou?
# os.makedirs(dataFolderPath, exist_ok=True)
channelsFilePath = os.path.abspath(os.path.join(dataFolder,'channelslist.json'))
#print("Ecriture dans : "+channelsFilePath) with open(channelsFilePath, "w") as file_write:
json.dump(datas, file_write)
with open(channelsFilePath, "w") as file_write: cohortecolorFilePath = os.path.abspath(os.path.join(dataFolder,'cohortescolor.json'))
json.dump(datas, file_write) with open(cohortecolorFilePath, "w") as file_write:
json.dump(cohortescolor, file_write)
cohortecolorFilePath = os.path.abspath(os.path.join(dataFolder,'cohortescolor.json')) cohorteFilePath = os.path.abspath(os.path.join(dataFolder,'cohorteslist.json'))
with open(cohortecolorFilePath, "w") as file_write: with open(cohorteFilePath, "w") as file_write:
json.dump(cohortescolor, file_write) json.dump(cohortes, file_write)
cohorteFilePath = os.path.abspath(os.path.join(dataFolder,'cohorteslist.json')) pprint("Nb displayed channels : " + str(nbChannels))
with open(cohorteFilePath, "w") as file_write: pprint("Nb cohorte channels : " + str(nbCohorte))
json.dump(cohortes, file_write)
pprint("Nb displayed channels : " + str(nbChannels)) if __name__ == "__main__":
pprint("Nb cohorte channels : " + str(nbCohorte)) main()
pprint("Nb total channels : " + str(totalChannels))

145
script/getstats.py
View file

@ -3,7 +3,7 @@
# toutes les chaines sont en unicode (même les docstrings) # toutes les chaines sont en unicode (même les docstrings)
from __future__ import unicode_literals from __future__ import unicode_literals
from pprint import pprint # from pprint import pprint
from rocketchat_API.rocketchat import RocketChat from rocketchat_API.rocketchat import RocketChat
import json import json
import dev_config as cfg import dev_config as cfg
@ -11,8 +11,7 @@ import os
import random import random
from datetime import datetime from datetime import datetime
from monthdelta import monthdelta from monthdelta import monthdelta
from common.channelhelper import getTsunamy from common.channelhelper import getTsunamy, Tsunami, getAllChannels
from common.channelhelper import Tsunami
def getColor(): def getColor():
r = random.randrange(255) r = random.randrange(255)
@ -38,19 +37,31 @@ def createElement(label, color, data) :
"data": data "data": data
} }
def setTsunamyInfo(tsunamy, messagesDataTsunamy, id, length):
if tsunamy & Tsunami.GLOBAL:
messagesDataTsunamy[Tsunami.GLOBAL][id] += length
if tsunamy & Tsunami.PROJECT:
messagesDataTsunamy[Tsunami.PROJECT][id] += length
if tsunamy & Tsunami.DEMOCRACY:
messagesDataTsunamy[Tsunami.DEMOCRACY][id] += length
if tsunamy & Tsunami.ECOLOGY:
messagesDataTsunamy[Tsunami.ECOLOGY][id] += length
if tsunamy & Tsunami.TECHNOLOGY:
messagesDataTsunamy[Tsunami.TECHNOLOGY][id] += length
def main(): def main():
rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'], rocket = RocketChat(cfg.rocket['user'], cfg.rocket['password'],
server_url='https://coa.crapaud-fou.org') server_url=cfg.rocket['server'])
index = 0
labels = [None] * 12 labels = [None] * 12
messagesByChannel = [] messagesByChannel = []
usersByChannel = [] usersByChannel = []
messagesDataTsunamy = { messagesDataTsunamy = {
"global": [0] * 12, Tsunami.GLOBAL: [0] * 12,
"project": [0] * 12, Tsunami.PROJECT: [0] * 12,
"democraty": [0] * 12, Tsunami.DEMOCRACY: [0] * 12,
"ecology": [0] * 12, Tsunami.ECOLOGY: [0] * 12,
"technology": [0] * 12, Tsunami.TECHNOLOGY: [0] * 12,
} }
usersGlobal = [] usersGlobal = []
@ -63,82 +74,70 @@ def main():
"messagesByChannel": messagesByChannel, "messagesByChannel": messagesByChannel,
"usersByChannel": usersByChannel, "usersByChannel": usersByChannel,
"messagesByTsunamy": [ "messagesByTsunamy": [
createElement("global", getColor(), messagesDataTsunamy['global']), createElement("global", getColor(), messagesDataTsunamy[Tsunami.GLOBAL]),
createElement("project", getColor(), messagesDataTsunamy['project']), createElement("project", getColor(), messagesDataTsunamy[Tsunami.PROJECT]),
createElement("democratie", getColor(), messagesDataTsunamy['democraty']), createElement("democratie", getColor(), messagesDataTsunamy[Tsunami.DEMOCRACY]),
createElement("ecologie", getColor(), messagesDataTsunamy['ecology']), createElement("ecologie", getColor(), messagesDataTsunamy[Tsunami.ECOLOGY]),
createElement("technologie", getColor(), messagesDataTsunamy['technology']) createElement("technologie", getColor(), messagesDataTsunamy[Tsunami.TECHNOLOGY])
], ],
"usersGlobal": usersGlobal "usersGlobal": usersGlobal
} }
usersTest = [None] * 12 uniqueUserGlobal = [None] * 12
while True: for channel in getAllChannels(rocket):
channels = rocket.channels_list(offset=index).json() dataMess = []
dataUsers = []
for channel in channels['channels']: print( channel['name'] )
dataMess = [] begin = date - monthdelta(12)
dataUsers = [] end = begin + monthdelta(1)
pprint( channel['name'] )
begin = date - monthdelta(12) tsunamy = getTsunamy(channel)
for id in range(0, 12):
if uniqueUserGlobal[id] == None:
uniqueUserGlobal[id] = []
labels[id] = begin.strftime("%b %Y")
print(f"\t{labels[id]}")
begindate = begin.isoformat()
enddate = end.isoformat()
resultMess = rocket.channels_history(channel['_id'], oldest= begindate, latest=enddate, count= 10000).json()
resultMess = list(filter(lambda mess: 't' not in mess, resultMess['messages']))
length = len(resultMess)
dataMess.append(length)
if length > 0:
setTsunamyInfo(tsunamy, messagesDataTsunamy, id, length)
users = []
for mess in resultMess:
users.append(mess['u']['_id'])
uniqueUserGlobal[id].append(mess['u']['_id'])
usersDistinct = set(users)
dataUsers.append(len(usersDistinct))
else:
dataUsers.append(0)
begin = end
end = begin + monthdelta(1) end = begin + monthdelta(1)
color = getColor()
messageByChannel = createElement(channel['name'], color,dataMess)
userByChannel = createElement( channel['name'], color,dataUsers)
tsunamy = getTsunamy(channel) messagesByChannel.append(messageByChannel)
usersByChannel.append(userByChannel)
for id in range(0, 12):
if usersTest[id] == None:
usersTest[id] = []
labels[id] = begin.strftime("%b %Y")
begindate = begin.isoformat()
enddate = end.isoformat()
resultMess = rocket.channels_history(channel['_id'], oldest= begindate, latest=enddate, count= 10000).json()
resultMess = list(filter(lambda mess: 't' not in mess, resultMess['messages']))
lenght = len(resultMess)
dataMess.append(lenght)
if lenght > 0:
if tsunamy & Tsunami.GLOBAL:
messagesDataTsunamy['global'][id] += lenght
if tsunamy & Tsunami.PROJECT:
messagesDataTsunamy['project'][id] += lenght
if tsunamy & Tsunami.DEMOCRACY:
messagesDataTsunamy['democraty'][id] += lenght
if tsunamy & Tsunami.ECOLOGY:
messagesDataTsunamy['ecology'][id] += lenght
if tsunamy & Tsunami.TECHNOLOGY:
messagesDataTsunamy['technology'][id] += lenght
users = []
for mess in resultMess:
users.append(mess['u']['_id'])
usersTest[id].append(mess['u']['_id'])
usersDistinct = set(users)
dataUsers.append(len(usersDistinct))
else:
dataUsers.append(0)
begin = end
end = begin + monthdelta(1)
color = getColor()
messageByChannel = createElement(channel['name'], color,dataMess)
userByChannel = createElement( channel['name'], color,dataUsers)
messagesByChannel.append(messageByChannel)
usersByChannel.append(userByChannel)
if channels['count'] + channels['offset'] >= channels['total']:
break
index += channels['count']
for id in range(0, 12): for id in range(0, 12):
usersTest[id] = len(set(usersTest[id])) uniqueUserGlobal[id] = len(set(uniqueUserGlobal[id]))
userGlobal = createElement( 'global', 'red', usersTest) userGlobal = createElement( 'global', 'red', uniqueUserGlobal)
usersGlobal.append(userGlobal) usersGlobal.append(userGlobal)
save(info) save(info)
if __name__ == "__main__": if __name__ == "__main__":
main() main()