прикреплённый файл:ejst.py к странице FrBrGeorge/PythonCopypasteProof

   1 #!/usr/bin/env python3
   2 '''
   3 '''
   4 from collections import Counter
   5 from dataclasses import dataclass, field
   6 from difflib import unified_diff, HtmlDiff
   7 from editdistance import eval as distance
   8 from itertools import chain
   9 from multiprocessing import Pool
  10 from pathlib import Path
  11 from pygments.formatters import Terminal256Formatter
  12 from pygments import highlight
  13 from pygments.lexers import PythonLexer, DiffLexer
  14 from pygments.styles.native import NativeStyle
  15 import ast
  16 import atexit
  17 import cmd
  18 import csv
  19 import getpass
  20 import gzip
  21 import os
  22 import pickle
  23 import re
  24 import readline
  25 import requests
  26 import shlex
  27 import shutil
  28 import sshtoken
  29 import subprocess
  30 import sys
  31 import time
  32 import tomli_w
  33 import tomllib
  34 import traceback
  35 
  36 CONTEST = "Python2022"
  37 reSPACEQ = re.compile(r'([\s"]+)')
  38 
  39 
  40 @dataclass
  41 class files:
  42     pwd: Path = Path().resolve()
  43     config: Path = pwd / ".ejst.conf"
  44     archive: Path = pwd / "var" / "archive" / "uuid"
  45     cache: Path = Path("~").expanduser() / ".cache" / CONTEST
  46     history: Path = cache / "history"
  47     token: Path = cache / "token"
  48     runs: Path = cache / "runs.csv"
  49     sourcefiles: Path = cache / "sources.toml"
  50     standings: Path = cache / "standings.html"
  51     distances: Path = cache / "distances.pickle"
  52     report: Path = cache / "report"
  53     diffdir: Path = cache / "diffs"
  54 
  55 
  56 @dataclass
  57 class _glob:
  58     allruns: list[dict] = field(default_factory=list)
  59     runs: list[dict] = field(default_factory=list)
  60     sourcefiles: dict[str] = field(default_factory=dict)
  61     standings: list[dict] = field(default_factory=list)
  62     users: dict[str] = field(default_factory=dict)
  63     scores: dict[str] = field(default_factory=dict)
  64     tasks: list[str] = field(default_factory=list)
  65     areas: dict[str] = field(default_factory=dict)
  66     logins: dict[str] = field(default_factory=dict)
  67     maxscore: int = 0
  68 
  69 
  70 glob = _glob()
  71 
  72 config = {
  73     "contest": f"{CONTEST}",
  74     "prompt": f"{CONTEST}> ",
  75     "moinroot": "/var/www/wiki/uneex",
  76     "moinpage": "data/pages/LecturesCMC(2f)PythonIntro2022(2f)HomeworkGradePaste(2f)report",
  77     "moinstatic": f"static/{CONTEST}",
  78     "moinlogin": "moin",
  79     "moinurl": "uneex.org",
  80     "login": "scoreviewer",
  81     "sshkey": "ED25519",
  82     "ejudge": "ejudge.cs.msu.ru",
  83     "ejuser": "frbrgeorge",
  84     "contest_id": "230",
  85     "min_score": 2 / 3,
  86     "max_distance": 0.01,
  87     "max_sizediff": 0.2,
  88     "common_tasks": ["HelloWorld", ],
  89     "agrade_tasks": ["InitParam", "TarFile", "DeStruct", "SpiralString"],
  90     "scores": [66, 77, 88],
  91     "common_downscore": 51,
  92     "same_downscore": 26,
  93     "paste_downscore": 0,
  94     "copypaste": {},
  95     "independent": {},
  96     "manual": {},
  97 }
  98 
  99 ejudge = {
 100     "interface": "new-master",
 101     "role": "1",
 102     "locale_id": "0",
 103     "stat": "94",
 104     "get": "152",
 105     "getxml": "155",
 106     "default": ("User_Name", "Run_Id", "Prob", "Status", "Score"),
 107     "cookies": None,
 108     "adm_url": None,
 109 }
 110 
 111 
 112 def ejget(action):
 113     if type(action) is str and action.isalpha():
 114         action = ejudge[action]
 115     url = re.sub(r"[&]action=\d+|$", f"&action={action}", ejudge["adm_url"])
 116     req = requests.get(url, cookies=ejudge["cookies"])
 117     return req.text
 118 
 119 
 120 def getruns():
 121     request = requests.post(f"https://{config['ejudge']}/{ejudge['interface']}", data={
 122         "login": config["login"],
 123         "password": sshtoken.get_token(files.token.as_posix()).decode(),
 124         "contest_id": config["contest_id"],
 125         "role": ejudge["role"],
 126         "locale_id": ejudge["locale_id"],
 127     })
 128     ejudge["cookies"], ejudge["adm_url"] = request.cookies, request.url
 129     return ejget("get"), ejget("stat")
 130 
 131 
 132 def auditpick(text, label):
 133     fmt = f"\n{label}: "
 134     return text[(st := text.index(fmt) + len(fmt)): text.index("\n", st + 1)]
 135 
 136 
 137 def getsourcefiles():
 138     sources = {}
 139     for path in files.archive.glob("??/??/*"):
 140         txt = (path / "audit").read_text()
 141         runid = auditpick(txt, "Run-id")
 142         if (path / "source").is_file():
 143             sources[runid] = (path / "source").as_posix()
 144         elif (path / "source.gz").is_file():
 145             sources[runid] = (path / "source.gz").as_posix()
 146     return sources
 147 
 148 
 149 AstK = ['None '] + [s for s in dir(ast) if s[0].isalpha()]
 150 AstD = {key: chr(char + 65) for key, char in zip(AstK, range(len(AstK)))}
 151 
 152 
 153 def astformat(node):
 154     if isinstance(node, ast.AST):
 155         args = []
 156         for fld in node._fields:
 157             value = getattr(node, fld)
 158             args.append(astformat(value))
 159         return '%s%s' % (AstD[node.__class__.__name__], ''.join(args))
 160     elif isinstance(node, list):
 161         return '%s' % ''.join(astformat(x) for x in node)
 162     return '@'
 163 
 164 
 165 def readsource(args):
 166     fname, user, task, runid = args
 167     if fname.endswith(".gz"):
 168         source = gzip.open(fname, "rb").read()
 169     else:
 170         source = open(fname, "rb").read()
 171     astree = ast.parse(source, "p")
 172     prep = astformat(astree)
 173     return user, task, ast.unparse(astree), prep, runid
 174 
 175 
 176 def readsources():
 177     minscore = glob.maxscore * config["min_score"]
 178     glob.users = {st["User"]: st for st in glob.standings if st["Score"] and int(st["Score"]) >= minscore}
 179     vals = []
 180     for run in glob.allruns:
 181         if run["Run_Id"] in glob.sourcefiles:
 182             if run["Stat_Short"] == "OK" and run["User_Name"] in glob.users:
 183                 vals.append((glob.sourcefiles[run["Run_Id"]], run["User_Name"], run["Prob"], run["Run_Id"]))
 184         else:
 185             print(f'ERROR: No {run["Run_Id"]} source, update is needed!')
 186     glob.scores, glob.areas = {}, {}
 187     with Pool() as pool:
 188         for user, task, source, prep, runid in pool.map(readsource, vals):
 189             if user:
 190                 glob.areas.setdefault(task, {})[user] = \
 191                     glob.scores.setdefault(user, {})[task] = \
 192                     [int(glob.users[user][task]), source, prep, set(), runid]
 193     for user in glob.scores:
 194         for task in glob.tasks:
 195             if task not in glob.scores[user]:
 196                 glob.scores[user][task] = [int(glob.users[user][task] or 0), "", "", set(), 0]
 197 
 198 
 199 def calcdist(args):
 200     return *args, distance(*args[-2:]) / (len(args[-1]) + len(args[-2]))
 201 
 202 
 203 def calcdistances():
 204     poolargs = []
 205 
 206     for task, area in glob.areas.items():
 207         if task in config["common_tasks"]:
 208             continue
 209         users = list(area.items())
 210         for i, (user, (score, source, prep, flags, runid)) in enumerate(users):
 211             if task in config["independent"].get(user, []):
 212                 continue
 213             for j in range(i + 1, len(users)):
 214                 user2, (score2, source2, prep2, flags2, runid2) = users[j]
 215                 if task in config["independent"].get(user2, []):
 216                     continue
 217                 w = abs(1 - (len(prep) + len(prep2)) / (2 * max(len(prep), len(prep2))))
 218                 if all((score, source, score2, source2, w < config["max_sizediff"])):
 219                     poolargs.append((task, user, user2, prep, prep2))
 220     print(f"Calculating {len(poolargs)} distances…", end=" ")
 221     with Pool() as pool:
 222         for task, user, user2, prep, prep2, dist in pool.map(calcdist, poolargs):
 223             if dist < config["max_distance"]:
 224                 glob.areas[task][user][3].add((dist, user2))
 225                 glob.areas[task][user2][3].add((dist, user))
 226     print("done!")
 227 
 228 
 229 def listtask(task):
 230     ranged = sorted((len(attrs[3]), min(attrs[3], default=(100, "")), user, attrs[0])
 231                     for user, attrs in glob.areas[task].items())
 232     weight = sum(r[0] for r in ranged)
 233     return weight, ranged
 234 
 235 
 236 def downscore():
 237     # TODO copypaste / independent
 238     counter = equal = 0
 239     for user, tasks in glob.scores.items():
 240         for task, attrs in tasks.items():
 241             if task in config["common_tasks"]:
 242                 attrs[0] = min(attrs[0], config["common_downscore"])
 243             if attrs[3]:
 244                 attrs[0] = min(attrs[0], config["same_downscore"])
 245                 counter += 1
 246                 for weight, buddy in attrs[3]:
 247                     if attrs[1] == glob.scores[buddy][task][1]:
 248                         equal += 1
 249                         attrs[0] = config["paste_downscore"]
 250                         break
 251 
 252     for user, tasks in config["copypaste"].items():
 253         for task in tasks:
 254             glob.scores[user][task][0] = config["paste_downscore"]
 255             for weight, buddy in glob.scores[user][task][3]:
 256                 glob.scores[buddy][task][0] = config["paste_downscore"]
 257     for user, tasks in config["manual"].items():
 258         for task, score in tasks.items():
 259             glob.scores[user][task][0] = score
 260     return counter, equal
 261 
 262 
 263 def select(where, *fields, **pattern):
 264     for run in where.values() if isinstance(where, dict) else where:
 265         if run | pattern == run:
 266             if fields:
 267                 yield {key: run[key] for key in fields}
 268             else:
 269                 yield run
 270 
 271 
 272 def showselect(where, *fields, **pattern):
 273     for run in select(where, *fields, **pattern):
 274         print(*run.values())
 275 
 276 
 277 def selectset(where, field, **pattern):
 278     return set(tuple(run.values())[0] for run in select(where, field, **pattern))
 279 
 280 
 281 def selectitem(where, *fields, error=True, **pattern):
 282     res = list(select(where, *fields, **pattern))
 283     if len(res) != 1:
 284         msg = f"Got {len(res)} selection of {pattern} instead of one"
 285         if error:
 286             raise ValueError(msg)
 287         print(msg, file=sys.stderr)
 288     if len(fields) == 1:
 289         return list(res[0].values())[0] if res else None
 290     else:
 291         return list(res[0].values()) if res else [None] * len(fields)
 292 
 293 
 294 def score(task, person):
 295     for run in select(glob.runs, "default", {"Prob": task, "User_Name": person}):
 296         print(*run.values())
 297 
 298 
 299 def readruns():
 300     glob.allruns = list(csv.DictReader(files.runs.open(newline=""), delimiter=";"))
 301     for run in glob.allruns:
 302         if not run["User_Name"]:
 303             run["User_Name"] = run["User_Login"]
 304         glob.logins[run["User_Name"]] = run["User_Login"]
 305     glob.tasks = list(selectset(glob.allruns, "Prob"))
 306     glob.maxscore = (len(glob.tasks) - len(config["common_tasks"])) * 100 + \
 307         len(config["common_tasks"]) * config["common_downscore"]
 308 
 309 
 310 def readstandings():
 311     standings = files.standings.read_text()
 312     htmltable = re.search(r'<table[^\n]*class="standings">(.*)</table>', standings, re.M | re.S).groups()[0]
 313     table = []
 314     for record in re.finditer(r'<tr[^>]+>(.*?)</tr>', htmltable, re.M | re.S):
 315         table.append([])
 316         for res in re.finditer(r'<t[dh][^>]+>(?:<[^>]+>)*(.*?)(?:<[^>]+>)*</t[dh]>', record.groups()[0]):
 317             val = res.groups()[0].replace('&nbsp;', '')
 318             table[-1].append(val)
 319     glob.standings = [dict(zip(table[0], table[i])) for i in range(1, len(table))]
 320 
 321 
 322 def gettoken():
 323     return sshtoken.get_token(files.token.as_posix(), config["sshkey"]).decode()
 324 
 325 
 326 def createtoken():
 327     sshtoken.store_token(getpass.getpass("Password: "), files.token.as_posix(), config["sshkey"])
 328 
 329 
 330 def dumpdistances():
 331     dists = {(user, task): attrs[3] for user, tasks in glob.scores.items() for task, attrs in tasks.items()}
 332     with files.distances.open("wb") as f:
 333         pickle.dump(dists, f)
 334 
 335 
 336 def loaddistances():
 337     with files.distances.open("rb") as f:
 338         dists = pickle.load(f)
 339     for (user, task), pasters in dists.items():
 340         glob.scores[user][task][3] |= pasters
 341 
 342 
 343 def grade(user):
 344     score = int(100 * sum(attrs[0] for attrs in glob.scores[user].values()) / glob.maxscore)
 345     solved = len([attrs[0] for attrs in glob.scores[user].values() if attrs[0] > config["paste_downscore"]])
 346     tasks = set(task for task, attrs in glob.scores[user].items() if attrs[0] > config["paste_downscore"])
 347     for gr, grname in zip(config["scores"] + [glob.maxscore + 1], ("НЕУД", "'''УДОВЛ'''", "'''ХОР'''", "'''ОТЛ'''")):
 348         if score < gr:
 349             break
 350 
 351     if grname == "'''ОТЛ'''" and not tasks.issuperset(set(config["agrade_tasks"])):
 352         grname = "'''ХОР+'''"
 353     passed = "'''ЗАЧЁТ'''" if solved == len(glob.tasks) or score >= config["scores"][1] else "незач"
 354     return grname, passed, int(score), int(solved)
 355 
 356 
 357 def makereport(commonmark="*"):
 358     def rep(*args):
 359         match args:
 360             case [nick, name]:
 361                 report.append(f"<<Anchor({nicks[nick]})>>[[#_{nicks[nick]}|back]]\n== {name} ==")
 362             case [nick, name, grad, passgr, score, solved]:
 363                 report.append(f"|| [[#{nicks[nick]}|{nick}]] || <<Anchor(_{nicks[nick]})>>{name}"
 364                               f" || {grad} || {passgr} || {score} || {solved} ||")
 365             case [user, task, score, weight, buddy]:
 366                 common = commonmark if task in config["common_tasks"] else ""
 367                 if buddy:
 368                     source, source2 = glob.scores[user][task][1], glob.scores[buddy][task][1]
 369                     runid, runid2 = glob.scores[user][task][4], glob.scores[buddy][task][4]
 370                     diffname = f"{task}_{runid}_{runid2}.diff.html"
 371                     attachments[diffname] = \
 372                         HtmlDiff().make_file(source.split('\n'), source2.split('\n'),
 373                                              f"{user} {task} {runid}", f"{buddy} {task} {runid2}")
 374                     diffurl = f"{config['moinstatic']}/report/{diffname}"
 375                     blink = f"[[#{nicks[glob.logins[buddy]]}|{buddy}]] '''([[{diffurl}|distance {weight}]])'''"
 376                 else:
 377                     blink = ""
 378                 report.append(f"|| `{task}`{common} || '''{score}''' || {blink} ||")
 379             case [str(text)]:
 380                 report.append(text)
 381             case _:
 382                 print("Unable to report", *args)
 383 
 384     nicks = {nick: nick.replace(" ", "_").replace("@", "-") for nick in glob.logins.values()}
 385     attachments = {}
 386     report = []
 387     agrade_tasks = " ".join(f"[[../../Homework_{t}|{t}]]" for t in config["agrade_tasks"])
 388     rep(f"""== Текущие лимиты ==
 389 Один балл таблицы — это {glob.maxscore/100} баллов EJudge.
 390 
 391 ||<|3> ⩾ {config['scores'][2]} || '''ОТЛ''' / '''ЗАЧЁТ''', если все обязательные задачи решены ||
 392 || '''ХОР+''' / '''ЗАЧЁТ''', если не решена хотя бы одна из обязательных задач: ||
 393 ||<)> {agrade_tasks} ||
 394 || ⩾ {config['scores'][1]} || '''ХОР''' / '''ЗАЧЁТ''' ||
 395 || ⩾ {config['scores'][0]} || '''УДОВЛ''' / незачёт ||
 396 || < {config['scores'][0]} || НЕУД / незачёт ||
 397 
 398 Про копипасту:
 399  * «Почти похожие» решения — не более s,,alike,,={config['same_downscore']}
 400  * Полностью совпадающие решения и подтверждённая копипаста — s,,equal,,={config['paste_downscore']}
 401  * «Слишком простые задачи» — не более s,,raw,,={config['common_downscore']} (отмечены «'''{commonmark}'''»))
 402 == {time.strftime("%F")} ==
 403 || '''User''' || '''Name ''' || '''Grade''' || '''Pass''' || '''Score''' || '''Solved''' ||""")
 404 
 405     for user in sorted(glob.users, key=lambda x: glob.logins[x]):
 406         rep(glob.logins[user], user, *grade(user))
 407     for user, tasks in glob.scores.items():
 408         rep(glob.logins[user], user)
 409         for task, attrs in tasks.items():
 410             rep(user, task, attrs[0], *min(attrs[3], default=(100500, "")))
 411         grname, passed, score, solved = grade(user)
 412         rep(f"|| {solved} / {len(glob.tasks)} || {score} || {grname}/{passed} ||")
 413 
 414     if files.report.is_dir():
 415         shutil.rmtree(files.report.as_posix())
 416     if files.diffdir.is_dir():
 417         shutil.rmtree(files.diffdir.as_posix())
 418     (files.diffdir).mkdir(parents=True)
 419     (files.report / "revisions").mkdir(parents=True)
 420     (files.report / "revisions" / "00000001").write_text("\n".join(report) + "\n")
 421     (files.report / "current").write_text("00000001\n")
 422     for name, html in attachments.items():
 423         (files.diffdir / name).write_text(html)
 424     return (files.report / "current").as_posix()
 425 
 426 
 427 def publish():
 428     remotmoin = f"{config['moinlogin']}@{config['moinurl']}:{config['moinroot']}/{config['moinpage']}"
 429     subprocess.check_output(["rsync", "-avP", "--delete", files.report.as_posix() + "/", remotmoin])
 430     remotestatic = f"{config['moinlogin']}@{config['moinurl']}:{config['moinroot']}/{config['moinstatic']}/report/"
 431     subprocess.check_output(["rsync", "-avP", "--delete", files.diffdir.as_posix() + "/", remotestatic])
 432 
 433 
 434 def init(sync=False):
 435     if not files.cache.is_dir():
 436         files.cache.mkdir()
 437     configread()
 438     if not files.token.is_file():
 439         print("WARNING: no password token is stored, creating one", file=sys.stderr)
 440         createtoken()
 441     if sync or not files.runs.is_file() or not files.standings.is_file():
 442         if not sync:
 443             print(f"WARNING: no {files.runs.name} is stored, downloading", file=sys.stderr)
 444         runs, standings = getruns()
 445         files.runs.write_text(runs)
 446         files.standings.write_text(standings)
 447     readruns()
 448     readstandings()
 449     if sync or not files.sourcefiles.is_file():
 450         tomli_w.dump(getsourcefiles(), files.sourcefiles.open("wb"))
 451     with files.sourcefiles.open("br") as f:
 452         glob.sourcefiles = tomllib.load(f)
 453     readsources()
 454     if sync or not files.distances.is_file():
 455         calcdistances()
 456         dumpdistances()
 457     loaddistances()
 458     print(*downscore(), "rewrites / copypastes in question")
 459 
 460 
 461 def configread():
 462     if not files.config.is_file():
 463         print(f"WARNING: no {files.config.as_posix()} file, creating default one")
 464         configwrite()
 465     with files.config.open("br") as f:
 466         config.update(tomllib.load(f))
 467 
 468 
 469 def configwrite():
 470     with files.config.open("bw") as f:
 471         tomli_w.dump(config, f)
 472 
 473 
 474 def taskdiff(task, user, user2):
 475     try:
 476         source, source2 = glob.areas[task][user][1], glob.areas[task][user2][1]
 477     except KeyError as E:
 478         print("Not found:", E)
 479         return
 480     source, source2 = glob.areas[task][user][1], glob.areas[task][user2][1]
 481     res = "\n".join(unified_diff(source.split("\n"), source2.split("\n"), user, user2, lineterm=''))
 482     print(highlight(res, DiffLexer(), Terminal256Formatter(style=NativeStyle)))
 483 
 484 
 485 def hyst(counter, title="", blocks=" ▁▂▃▄▅▆▇█", height=None):
 486     mn, mx = min(counter), max(counter)
 487     Mn, Mx = min(counter.values()), max(counter.values())
 488     if title:
 489         print("   ", title.center(mx - mn))
 490     if height is None:
 491         height = Mx - Mn + 1
 492     for i in range(height, -1, -1):
 493         print(f"{round(Mn + i * (Mx - Mn) / height):>3}: ", end="")
 494         for j in range(mn, mx + 1):
 495             h = counter[j] * height / (Mx - Mn)
 496             c = blocks[0] if h < i else (blocks[-1] if h >= i + 1 else blocks[round((h - i) * (len(blocks)))])
 497             print(c, end="")
 498         print()
 499     print("    ", mn, " " * (mx - mn - len(f"{mx} {mn}")), mx)
 500 
 501 
 502 def isnumbers(*strings):
 503     for string in strings:
 504         try:
 505             res = float(string)
 506         except ValueError:
 507             res = None
 508             yield res
 509         else:
 510             yield int(res) if res.is_integer() else res
 511 
 512 
 513 class Shell(cmd.Cmd):
 514     prompt = config["prompt"]
 515 
 516     @staticmethod
 517     def shplit(string):
 518         try:
 519             return shlex.split(string)
 520         except ValueError as E:
 521             print(E)
 522             return string.split()
 523 
 524     @staticmethod
 525     def match(text, select, *add, shift=0, prefix=""):
 526         vals = tuple(filter(None, select))
 527         return [prefix + key[shift:] for key in vals + add if key[shift:] and key.startswith(text.strip('"'))]
 528 
 529     @staticmethod
 530     def locateword(text, line, begidx, endidx):
 531         s = line[:endidx]
 532         if s.count('"') % 2:
 533             s += '"'
 534         return len(shlex.split(s)) - (s[-1] != " ")
 535 
 536     @staticmethod
 537     def getword(text, line, begidx, endidx):
 538         s = line[:endidx]
 539         if s.count('"') % 2:
 540             s += '"'
 541             return shlex.split(s)[-1]
 542         return text
 543 
 544     def do_config(self, arg):
 545         '''Configuration
 546 \t\tShow all config parameters
 547 <param>\t\tShow config parameter
 548 <param>\t<value>\tSet config parameter
 549 token\t\tSet password silently from stdin
 550         '''
 551         match self.shplit(arg):
 552             case []:
 553                 for cname in config:
 554                     print(f"\t{cname}:\t{config[cname]}")
 555             case ["token"]:
 556                 createtoken()
 557             case [param]:
 558                 if param.startswith("-"):
 559                     if param[1:] in config:
 560                         del config[param[1:]]
 561                 elif param in config:
 562                     print(f"{config[param]}")
 563                 else:
 564                     print(f"ERROR: unknown '{param}' parameter")
 565             case [param, *values]:
 566                 if param.startswith("!"):
 567                     param = param[1:]
 568                 elif param not in config:
 569                     print(f"ERROR: unknown '{param}' parameter")
 570                 numbers = tuple(isnumbers(*values))
 571                 if None in numbers:
 572                     if isinstance(config[param], list):
 573                         config[param] = values
 574                     else:
 575                         config[param] = shlex.join(values)
 576                 else:
 577                     config[param] = numbers if len(values) > 1 else numbers[0]
 578 
 579     def complete_config(self, text, line, begidx, endidx):
 580         match self.locateword(text, line, begidx, endidx):
 581             case 1:
 582                 return self.match(text, config, "token")
 583             case 2:
 584                 res = config.get(line.split()[1], "UNKNOWN")
 585                 return [" ".join(res) if isinstance(res, list) else res]
 586 
 587     def do_write(self, arg):
 588         '''Write some changed data'''
 589         match arg:
 590             case "config":
 591                 configwrite()
 592 
 593     def complete_write(self, text, line, begidx, endidx):
 594         fnames = ["config"]
 595         return self.match(text, fnames)
 596 
 597     def do_task(self, arg):
 598         '''List task statistics
 599 
 600 \t\tList all tasks
 601 task\t\tShow task statistic
 602 task user\tShow user's task details
 603 task user user2\tCompare task solutions
 604         '''
 605         if not arg:
 606             res = sorted((listtask(task), task) for task in glob.tasks)
 607             for (weight, ranged), task in res:
 608                 print(f"{task}: {len(ranged)} total, copypaste weight {weight}")
 609         match self.shplit(arg):
 610             case [task]:
 611                 if task in glob.tasks:
 612                     weight, ranged = listtask(task)
 613                     for total, dist, user, score in ranged:
 614                         print(f"{user}: total {total}, min {dist} / score {score}")
 615                     print(f"Copypaste weight {weight}")
 616             case [task, user]:
 617                 if task in glob.tasks and user in glob.users:
 618                     for dist, paster in glob.areas[task][user][3]:
 619                         print(f"{paster}: {dist}")
 620                     print(highlight(glob.areas[task][user][1], PythonLexer(), Terminal256Formatter(style=NativeStyle)))
 621             case [task, user, user2]:
 622                 taskdiff(task, user, user2)
 623 
 624     def complete_task(self, text, line, begidx, endidx):
 625         match self.locateword(text, line, begidx, endidx):
 626             case 1:
 627                 return self.match(text, glob.tasks)
 628             case _:
 629                 w = self.getword(text, line, begidx, endidx)
 630                 return self.match(w, glob.users, shift=len(w) - len(text))
 631 
 632     def do_manual(self, arg):
 633         '''Alias for 'score manual'''
 634         self.do_score("manual")
 635 
 636     def do_score(self, arg):
 637         '''Show user scores
 638 
 639 \t\tShow all users' scores
 640 'manual'\t\tShow manual defined scores
 641 user\t\tShow all user scores
 642 user task\tShow detailed user task score
 643 user task score\tSet manual user task score
 644 user task -\tClear manual user task score
 645 user task user2\tCompare solutions
 646         '''
 647         if not arg:     # TODO calculate
 648             for user, score in glob.users.items():
 649                 print(f"{user}: {score['Score']}")
 650             return
 651         user, *tasks = self.shplit(arg)
 652         if user == "manual":
 653             for u in config["manual"]:
 654                 for task, score in config["manual"][u].items():
 655                     print(f"{u} / {task} = {score}")
 656             return
 657         match tasks:
 658             case []:
 659                 print(f"\t{user}")
 660                 res = [f"{task}: {score[0]}" for task, score in glob.scores[user].items()]
 661                 self.columnize(sorted(res))
 662                 s = sum(int(score[0]) for score in glob.scores[user].values())
 663                 print(f"\t{s}/{glob.users[user]['Score']}")
 664                 if user in glob.scores:
 665                     print("Result:", *grade(user))
 666             case ["manual"]:
 667                 for task, score in config["manual"][user].items():
 668                     print(f"{user} / {task} = {score}")
 669             case [task]:
 670                 showselect(glob.allruns, *ejudge["default"], Prob=task, User_Name=user)
 671                 print(f"Run: {glob.users[user][task]}")
 672                 for dist, paster in glob.scores[user][task][3]:
 673                     print(f"{paster}: {dist}")
 674                 print("Score:", glob.scores[user][task][0])
 675             case [task, score] if score.isdigit():
 676                 config["manual"].setdefault(user, {})[task] = int(score)
 677                 if int(score) > config["same_downscore"]:
 678                     config["independent"].setdefault(user, []).append(task)
 679             case [task, "-"]:
 680                 if task in config["manual"].get(user, {}):
 681                     del config["manual"][user][task]
 682                     downscore()
 683             case [task, user2]:
 684                 taskdiff(task, user, user2)
 685 
 686     def complete_score(self, text, line, begidx, endidx):
 687         match self.locateword(text, line, begidx, endidx):
 688             case 1:
 689                 w = self.getword(text, line, begidx, endidx)
 690                 return self.match(w, glob.users, "manual", shift=len(w) - len(text))
 691             case 2:
 692                 return self.match(text, glob.tasks, "manual")
 693             case 3:
 694                 w = self.getword(text, line, begidx, endidx)
 695                 return self.match(w, glob.users, shift=len(w) - len(text))
 696 
 697     def do_stat(self, arg):
 698         """Show statistices"""
 699         print("Tasks by number of solutions:")
 700         ts = [f"{t}: {s}" for t, s in sorted((len(users), task) for task, users in glob.areas.items())]
 701         self.columnize(ts)
 702         print(f"Users: {len(glob.users)} / accounted {len(glob.scores)}")
 703         grades = Counter(grade(user)[0].strip("'") for user in glob.scores)
 704         passes = Counter(grade(user)[1].strip("'") for user in glob.scores)
 705         scores = Counter(grade(user)[2] for user in glob.scores)
 706         solved = Counter(grade(user)[3] for user in glob.scores)
 707         print(", ".join(f"{k}:{v}" for k, v in grades.items()))
 708         print(", ".join(f"{k}:{v}" for k, v in passes.items()))
 709         hyst(solved, title="Solved", height=8)
 710         hyst(scores, title="Scores", height=8)
 711         grsl = [(grade(user)[0].strip("'"), grade(user)[3]) for user in glob.scores]
 712         for gr in 'ОТЛ', 'ХОР+', 'ХОР', 'УДОВЛ':
 713             m = min((g, sl) for g, sl in grsl if g == gr)
 714             M = max((g, sl) for g, sl in grsl if g == gr)
 715             print(f"{gr}: min {m} / max {M}")
 716 
 717     def do_select(self, arg):
 718         '''[internal] Browse list of dicts'''
 719         obj, *selectors = self.shplit(arg)
 720         names = [par for par in selectors if "=" not in par]
 721         if names == ["default"]:
 722             names = ejudge["default"]
 723         filters = dict(par.split("=") for par in selectors if "=" in par)
 724         if len(names) == 1:
 725             print(*sorted(selectset(getattr(glob, obj), *names, **filters)))
 726         else:
 727             showselect(getattr(glob, obj), *names, **filters)
 728 
 729     def complete_select(self, text, line, begidx, endidx):
 730         if self.locateword(text, line, begidx, endidx) == 1:
 731             return self.match(text, glob.__dataclass_fields__.keys())
 732         if "=" not in line.split()[-1] or text == "" and line[begidx - 1] != "=":
 733             suffix = "=" in line and "=" or ""
 734             where = getattr(glob, line.split()[1])
 735             words = next(iter(where.values())) if isinstance(where, dict) else where[0]
 736             return [s + suffix for s in self.match(text, words, "default")]
 737         root, (field, val) = line.split()[1], line.split()[-1].split("=")
 738         variants = sorted(selectset(getattr(glob, root), field))
 739         return self.match(val, variants)
 740 
 741     def do_copypaste(self, arg):
 742         '''Display or register user+task as proved copypaste
 743 
 744 ?\t\tDisplay all useras with close enough solutions
 745 \t\tDisplay all proved copypaste
 746 user\t\tDisplay user's proved copypaste
 747 user ?\t\tDisplay user's close enough solutions
 748 user task\tRegister task as copypasted by user
 749 user -task\tUnrgister task as copypasted by user
 750         '''
 751         self._do_user_tasks(arg, "copypaste")
 752 
 753     def _do_user_tasks(self, arg, table):
 754         if not arg:
 755             for user, tasks in config[table].items():
 756                 print(f"{user}:", *tasks)
 757             return
 758         match self.shplit(arg):
 759             case ["?"]:
 760                 plist = sorted((sum(len(a[3]) for a in tasks.values()), user) for user, tasks in glob.scores.items())
 761                 for pastes, user in plist:
 762                     print(f"{user}: {pastes} / {grade(user)[0]}")
 763             case [user]:
 764                 if user in glob.users:
 765                     print(*config[table].get(user, []))
 766             case [user, "?"]:
 767                 for task, attrs in glob.scores[user].items():
 768                     for weight, buddy in attrs[3]:
 769                         print(f"{task}: {weight} = {buddy}")
 770             case [user, str(task)] if task.startswith("-"):
 771                 if user in glob.users and task[1:] in config[table].get(user, []):
 772                     config[table][user].remove(task[1:])
 773                     if config[table][user] == []:
 774                         del config[table][user]
 775             case [user, str(task)]:
 776                 if user in glob.users and task not in config[table].setdefault(user, []):
 777                     config[table][user].append(task)
 778         downscore()
 779 
 780     def complete_copypaste(self, text, line, begidx, endidx):
 781         match self.locateword(text, line, begidx, endidx):
 782             case 1:
 783                 w = self.getword(text, line, begidx, endidx)
 784                 return self.match(w, glob.users, shift=len(w) - len(text))
 785             case 2:
 786                 return self.match(text, glob.tasks)
 787 
 788     def do_independent(self, arg):
 789         '''Display or register user+task as proved independent
 790 
 791 \t\tDisplay all tasks proved independent
 792 user\t\tDisplay user's  tasks proved independent
 793 user task\tRegister task as independent by user
 794 user -task\tUnrgister task as independent by user
 795         '''
 796         self._do_user_tasks(arg, "independent")
 797 
 798     def complete_independent(self, text, line, begidx, endidx):
 799         return self.complete_copypaste(text, line, begidx, endidx)
 800 
 801     def do_report(self, arg):
 802         '''Create a Moin report with HTML attachments. Use report publish for pablish also'''
 803         print(makereport())
 804         if arg.endswith("publish"):
 805             publish()
 806 
 807     def complete_report(self, text, line, begidx, endidx):
 808         return ["publish"]
 809 
 810     def do_sync(self, arg):
 811         '''Synchronize statistics and git'''
 812         remote = f"cd ~/{config['contest']} && git pull &&"\
 813                  "git add . && git commit -a -m ejst && git push && git pull || :"
 814         subprocess.check_output(["ssh", "-q", f"{config['ejuser']}@{config['ejudge']}", f"{remote}"], stdin=None)
 815         subprocess.check_output(["git", "pull"])
 816         init(sync=True)
 817 
 818     def do_search(self, arg):
 819         '''Search for something
 820 
 821 string\tSearch for user name or user login
 822 number\tSearch for run number
 823         '''
 824         match self.shplit(arg):
 825             case [str(substr)] if substr.isdigit():
 826                 prob, status, name = selectitem(glob.allruns, "User_Name", "Prob", "Status", error=False, Run_Id=substr)
 827                 if name:
 828                     print(f"{substr}: {prob} / {status} / {name}")
 829             case [str(substr)]:
 830                 for user, login in glob.logins.items():
 831                     if substr.lower() in user.lower() or substr.lower() in login.lower():
 832                         print(f"{user}: {login}")
 833 
 834     def complete_search(self, text, line, begidx, endidx):
 835         if text.isdigit():
 836             runids = (str(glob.scores[user][task][4]) for user in glob.users for task in glob.tasks)
 837             return self.match(text, runids)
 838         elif text:
 839             return [w for w in chain(*glob.logins.items()) if text.lower() in w.lower()]
 840 
 841     def do_eval(self, arg):
 842         print(eval(arg))
 843 
 844     def complete_eval(self, text, line, begidx, endidx):
 845         if (dot := text.rfind(".")) > 0:
 846             return self.match(text[dot + 1:], dir(eval(text[:dot])), prefix=text[:dot + 1])
 847         else:
 848             return self.match(text, globals())
 849 
 850     def do_shell(self, arg):
 851         """Alias of "! [command [arguments]]". Run shell or execute command"""
 852         if arg:
 853             res = subprocess.run(self.shplit(arg), stdin=sys.stdin, capture_output=True)
 854             print(res.stdout.decode())
 855             if res.stderr:
 856                 print(res.stderr.decode())
 857         else:
 858             subprocess.call([os.environ["SHELL"], "-l"])
 859 
 860     def do_EOF(self, arg):
 861         """Press Ctrl+D to exit"""
 862         return True
 863 
 864     def emptyline(self):
 865         pass
 866 
 867     def preloop(self):
 868         if files.history.is_file():
 869             readline.read_history_file(files.history.as_posix())
 870             readline.set_history_length(1000)
 871         atexit.register(readline.write_history_file, files.history.as_posix())
 872 
 873 
 874 def main():
 875     init()
 876     for errors in range(100):
 877         try:
 878             Shell().cmdloop()
 879         except Exception as E:
 880             traceback.print_tb(E.__traceback__)
 881             print(f"[{errors}] {type(E).__name__}: {E}", file=sys.stderr)
 882         except KeyboardInterrupt:
 883             print("^C", file=sys.stderr)
 884         else:
 885             break
 886     else:
 887         print("Too many exceptions, exiting", file=sys.stderr)
 888 
 889 
 890 if __name__ == "__main__":
 891     main()