Commit 79902a74 authored by Carsten Eie Frigaard's avatar Carsten Eie Frigaard
Browse files

update

parent 103cd9ec
......@@ -104,8 +104,9 @@ rækkefølge:}
\li{[OPTIONAL] ekstra, mest for de matematik intereserede: [BR] \indent{\ipynb{math_differential_calculus.ipynb} [GITHOML].}}
}
\p{Pandas er et meget populært databehandlingsværktøj, men vi kommer ikke til at
bruge Pandas i dette kursus.}
\p{Pandas er et meget populært databehandlingsværktøj, men det
bruges/introduceres dog ikke formelt i dette kursus (du er velkommen til selv
at undersøg/bruge det).}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
......
COUSEPLAN
HEAD
Uge{40} |Dato{110} |Lektion{20}|Emne{200} |Opgave{90} |Kommentar{150}
CONTENT
05 |02/02-2021 |L01 |Intro | |
06 |09/02-2021 |L02 |Klassifikation | |
07 |16/02-2021 |L03 |End-to-end ML | |
08 |23/02-2021 |L04 |Regression |O1 (28/02) |
09 |02/03-2021 |L05 |Data analyse | |
10 |09/03-2021 |L06 |Neurale netværk (NN) | |
11 |16/03-2021 |L07 |Træning og generalisering |O2 (17/03) |
12 |23/03-2021 |L08 |Regularisering og søgning | |
13 |30/03-2021 | | | |Påske (ingen undervisning)
14 |06/04-2021 |L09 |Deep learning (CNN) | |
15 |13/04-2021 |L10 |Probabilistiske modeller |O3 (14/04) |
16 |20/04-2021 |L11 |Unsupervised learning I (PCA) | |
17 |27/04-2021 |L12 |Unsupervised learnindg II (Kmeans,GMM) | |Kursus-evaluering
18 |04/05-2021 |L13 |Projektarbejde (Discord) | |
19 |11/05-2021 |L14 |O4 projekt | |
20 |18/05-2021 |L15 |O4 projekt |O4 (19/05) |
REFS
[1] sdf
[2] sfd zdf
END
\ No newline at end of file
#PYFILE = $(wildcard *.py) $(wildcard Utils/*.py)
PYCOURSE = mk_course.py
PYFILETREE = mk_filetree.py
PYPLAN = mk_plan.py
PWD = $(shell pwd)
COURSEFILES = ITMAL_E21
MKDIFF=colordiff -dw Refs/$1 $1 && echo "DIFF OK" && rm $1
build:
@ $(PYCOURSE) -c $(COURSEFILES)/course.tex -v -o ../../Html/
@ $(PYPLAN) -p $(COURSEFILES)/plan.txt -v -o ../../Html/plan.html
@ cd ../.. && $ $(PWD)/$(PYFILETREE) -v -o Html/filetree.html
TESTARGS=-v -v
test:
@ $(PYCOURSE) -c Test/course.tex $(TESTARGS) -o ./
@ $(call MKDIFF,L00.html)
@ $(call MKDIFF,L01.html)
@ $(PYPLAN) -p Test/plan.txt $(TESTARGS) -o plan.html
@ $(call MKDIFF,plan.html)
@ cd Test && $(PWD)/$(PYFILETREE) $(TESTARGS) -o $(PWD)/tree.html
@ $(call MKDIFF,tree.html)
sync: build
cd .. && scp -r Html/*.html si:ITMAL_E21/Html
ssh si "cd ITMAL_E21 && git pull"
edit:
joe $(PYCOURSE)
clean:
@ rm -f index.html out.html test.html ../Html/L??.html
#get:
# wget https://gitlab.au.dk/au204573/brightspacemods/-/raw/master/JS/demo.js
#
#sshfs:
# echo "not implemented yet"
#
#ae:
# sudo tail -f /var/log/apache2/error.log
......@@ -360,7 +360,7 @@ if __name__ == '__main__':
verbose = isInt(args.v)
coursefile = isStr(args.c)
Dbg(verbose, f"{Col('PURPLE')}GENERATING course file '{coursefile}'..{ColEnd()}")
Dbg(verbose, f"{Col('PURPLE')}GENERATING html course from file '{coursefile}'..{ColEnd()}")
htmlencoded = [HtmlEncode(i) for i in LoadCourseFile(coursefile)]
htmlstructure = ParseStructure(htmlencoded)
......
#!/usr/bin/env python3
from Utils.dbg import ERR, WARN
from Utils.colors import Col, ColEnd
from Utils.mkutils import *
from sys import argv, stderr
from os import listdir
from os.path import join, isfile, isdir
from argparse import ArgumentParser
from urllib.request import urlretrieve
if __name__ == '__main__':
def Find(root, excludepat):
assert isinstance(root, str)
assert isinstance(excludepat, list)
tree = {}
tree['trees'] = []
tree['files'] = []
for i in listdir(root):
if i is not None:
assert isinstance(i, str)
d = join(root, i)
assert isinstance(d, str)
if d[0:2]=="./":
d = d[2:]
skip = False
for j in excludepat:
if d.find(j)>=0:
skip = True
break
if not skip:
isDir = isdir(d)
t = 'trees' if isDir else 'files'
tree[t].append((i, Find(d, excludepat) if isDir else d))
return tree
def PrintItem(i, level, isDir):
def Tab(level, tab):
assert isinstance(level, int) and level>=0
assert isinstance(tab, str)
assert isinstance(isDir, bool)
#if htmlmode:
# return "", ""
# #return f"<p style='padding-left: {(level+1)*40}px;'>", "</p>"
t = ""
for n in range(level):
t += tab
return t, ""
def Newline():
return "<br>" if htmlmode else ""
def Link(i, isDir):
def CheckUrl(linkurl):
assert isinstance(linkurl, str)
assert linkurl.find("'") < 0
if testurls:
sfx = linkurl.split(".")[-1]
if sfx!="py":
#print(f"WEB GET test {linkurl}..")
try:
# wget: download(linkurl, bar=None)
urlretrieve(linkurl, filename=None)
except Exception as ex:
#PrettyPrintTracebackDiagnostics(ex)
print(f"{Col('LRED UNDERLINE')}EXCEPTION:{ColEnd()}{Col('LRED')} {ex}{ColEnd()}", stderr)
WARN(f"ignore test on link {linkurl}")
return linkurl
assert isinstance(url, str)
assert isinstance(isDir, bool)
assert isTuple(i)
if isDir:
return i[0]
assert isinstance(i[0], str)
assert isinstance(i[1], str)
linkurl = CheckUrl(url + "/" + i[1])
assert linkurl.find("'") < 0
r = f"<a href='{linkurl}'>{i[0]}</a>"
return r
assert isTuple(i)
assert isinstance(i[0], str)
nbsp = "&nbsp;"
tab = Tab(4*level, nbsp if htmlmode else " ")
link = Link(i, isDir) if htmlmode else i[0]
#pre = "<code>" if htmlmode else ""
#post = "</code>" if htmlmode else ""
pre = f"<span style=\"font-family: 'courier new', courier, sans-serif\">" if htmlmode else ""
post = "</span>" if htmlmode else ""
assert isTuple(tab)
r = pre + tab[0] + link + tab[1] + Newline() + post
Print(r, outputfile)
return r
def PrintTree(tree, level=0):
assert isinstance(tree, dict)
assert isinstance(url, str)
assert isinstance(htmlmode, bool)
files=0
dirs=0
for j in ['trees', 'files']:
t = tree[j]
assert isinstance(t, list)
d = sorted(tree[j])
assert isinstance(d, list)
for i in d:
assert isTuple(i)
assert isinstance(i[1], dict) == (j=='trees')
PrintItem(i, level, isinstance(i[1], dict))
files += 1
if j=='trees':
r = isTuple(PrintTree(i[1], level+1))
files += r[0]
dirs += 1 + r[1]
return (files, dirs)
try:
# https://gitlab.au.dk/au204573/itmal/-/raw/master/L01/modules_and_classes.ipynb
#url = "https://gitlab.au.dk/au204573/itmal/-/raw/master"
verbose = 0
url = "https://itundervisning.ase.au.dk/ITMAL_E21"
welcome = "ITMAL Backend File Structure"
htmlmode = True
testurls = False
outputfile = None
parser = ArgumentParser(prog=argv[0], description="", epilog="version 0.1")
parser.add_argument("-v", default=verbose, action="count", help=f"increase output verbosity, default={verbose}\n")
parser.add_argument("-t", default=testurls, action="store_true", help=f"test link by url download, default={testurls}\n")
parser.add_argument("-plain", default=not htmlmode, action="store_true", help=f"output in plain mode, default={not htmlmode}\n")
parser.add_argument("-url", default=url, type=str, help=f"url to prefix with (ending in '/'), default='{url}'\n")
parser.add_argument("-o", default=outputfile, type=str, help=f"output to this file (stdout==None), default='{outputfile}'\n")
args = parser.parse_args()
verbose = isInt(args.v)
htmlmode = not args.plain
assert isinstance(htmlmode, bool)
testurls = args.t
assert isinstance(testurls, bool)
url = args.url
assert isinstance(url, str)
if url[-1]=='/':
ERR("no trailing '/' in url, please add")
root = "./"
Dbg(verbose, f"{Col('PURPLE')}GENERATING html file tree from root '{root}'" + (f", (url='{url}')" if verbose > 0 else "") + f"..{ColEnd()}")
tree = Find(root, ["git", "Old", "ipynb_checkpoints", "~", "__pycache__"])
outputfile = Outputfile(args.o)
if htmlmode:
Print("<!DOCTYPE html>\n<html>\n<body>\n", outputfile)
Print("<h3>" + welcome + "</h3>\n", outputfile)
(files, dirs) = PrintTree(tree)
Dbg(verbose, f" {Col('YELLOW')}FOUND files={files}, dirs={dirs}{ColEnd()}")
if htmlmode:
Print("\n</body></html>", outputfile)
Dbg(verbose, f"{Col('PURPLE')}DONE{ColEnd()}")
except Exception as ex:
HandleException(ex)
#!/usr/bin/env python3
from Utils.dbg import ERR
from Utils.colors import Col, ColEnd
from Utils.mkutils import *
from sys import argv
from argparse import ArgumentParser
from html import escape
if __name__ == '__main__':
verbose = 0
planfile = "plan.txt"
outputfile= None
def ParseStructure(planlist):
def ParseWidths(headers):
w = []
h = []
for i in isList(headers):
curr = 65
if i[-1]=="}":
n = i.find("{")
if n>=0:
curr = int(i[n+1:-1])
#print(curr)
h.append(i[:n])
else:
h.append(i)
w.append(isInt(curr, 0))
assert len(w)==len(headers) and len(w)==len(h)
return w, h
def ParseLine(h, expectedlen):
s = isStr(h).split("|")
N = len(s)
if N != expectedlen:
if expectedlen < 0 and N < 2:
ERR(f"expected more than one column in header '{h}', but got {N} column(s)")
elif expectedlen >= 0:
ERR(f"expected exaclty {expectedlen} column(s) in line '{h}', but got {N} column(s)")
r = []
o = ""
for i in s:
t = Trim(i)
o += (" " if expectedlen < 0 else " ") + t + ("\n" if expectedlen < 0 else " | ")
r.append(t)
Dbg(verbose, str(o), 3)
return r
def CheckLine(i, expected):
Check(i<len(planlist), "i={i} < len of planlist={len(planlist}")
Check(planlist[i]==expected, f"planlist missing tag '{expected}' at line {i}, got='{planlist[i]}'")
N = len(isList(planlist))
if N < 7:
ERR("planlist too short, expected at least three lines with COUSEPLAN/HEAD/CONTENT/REFS/END")
CheckLine(0, "COUSEPLAN")
CheckLine(1, "HEAD")
CheckLine(3, "CONTENT")
headers = ParseLine(planlist[2], -1)
widths, headers = ParseWidths(headers)
M = len(headers)
s = {}
s['headers'] = headers
s['widths'] = widths
curr = []
for j in range(4, N):
i = isStr(Trim(planlist[j]))
if i=="REFS":
s['content']=curr
curr = []
if i=="END":
e = True
s['refs']=curr
curr = None
break
curr.append(i)
Check(curr is None, "plainlist missing END tag")
Check(s.get('content'), "planlist missing content structure")
Check(s.get('refs'), "planlist missing ref structure")
parsed = []
for i in s['content']:
l = ParseLine(i, M)
Dbg(verbose, str(l), 2)
parsed.append(l)
s['parsed'] = parsed
return s
def GenerateHtml(headers, widths, parsed, fullhtmldoc):
def HtmlEncode(s):
return escape(isStr(s, False))
M = len(isList(headers))
html = '<!DOCTYPE html>\n<html>\n<body>\n' if isBool(fullhtmldoc) else ""
html += '<table cellspacing="0px" cellpadding="1px" border="1px" align="center">\n<tbody>\n<tr style="background-color: #000000;" align="center">\n'
for i in zip(isList(headers), isList(widths)):
isTuple(i)
html += f'<td width="{isInt(i[1])}"><span style="background-color: #000000; color: #ffffff;"><strong>{HtmlEncode(i[0])}</strong></span></td>\n'
html += '</tr>\n'
for j in isList(parsed):
assert len(isList(j))==M
html += '<tr align="center">\n'
assert len(j)==len(widths)
for k in zip(j, widths):
isTuple(k)
html += f'<td width="{isInt(k[1])}">{HtmlEncode(k[0])}</td>\n'
html += '</tr>'
html += '</table>\n'
html += '</body>\n</html>' if isBool(fullhtmldoc) else ""
Dbg(verbose, f" {Col('YELLOW')}FOUND a {len(parsed)} x {len(headers)} table (rows x columns){ColEnd()}")
return html
try:
planfile = "plan.txt"
parser = ArgumentParser(prog=argv[0], epilog="version 0.1")
parser.add_argument("-v", default=verbose, action="count", help= "increase output verbosity, default={verbose}\n")
parser.add_argument("-t", default=False, action="store_true", help=f"generate simple table (witouth <html> <body> etc tags), default=False\n")
parser.add_argument("-p", default=planfile, type=str, help=f"planfile, default='{planfile}'\n")
parser.add_argument("-o", default=outputfile, type=str, help=f"outputfilt, default='{outputfile}\n")
args = parser.parse_args()
verbose = isInt(args.v)
planfile = args.p
Dbg(verbose, f"{Col('PURPLE')}GENERATING html plan from file '{planfile}..{ColEnd()}")
structure = ParseStructure(LoadText(planfile))
Dbg(verbose, str(structure['headers']), 2)
html = GenerateHtml(structure['headers'], structure['widths'], structure['parsed'], not args.t)
with Outputfile(args.o) as f:
f.write(html)
Dbg(verbose, f"{Col('PURPLE')}DONE{ColEnd()}")
except Exception as ex:
HandleException(ex)
\ No newline at end of file
PYCOURSE = mk_course.py
PYFILETREE = mk_htlm_filetree.py
PYPLAN = mk_course_plan.py
build:
@ $(PYCOURSE) -v -o ../Html/
@ cd .. && $ Etc/$(PYFILETREE) -v -o Html/filetree.html
@ $(PYPLAN) -v -o ../Html/plan.html
test:
@ $(PYPLAN) -v -v -o index.html
@ colordiff -dw Refs/plan.ref index.html
sync: build
cd .. && scp -r Html/*.html si:ITMAL_E21/Html
ssh si "cd ITMAL_E21 && git pull"
edit:
joe $(PYCOURSE)
clean:
@ rm -f out.html ../Html/L??.html
#JSFILES = $(wildcard JS/*.js)
#PYFILE = $(wildcard Py/*.py) $(wildcard Py/Utils/*.py)
#PYFILETREE = Py/mk_htlm_filetree.py
#PYPLAN = Py/mk_course_plan.py
#TESTFILETREEDIR = ~/ASE/ITMAL/itmal
#PWD = $(shell pwd)
#testplan:
# @ $(PYPLAN) -v -v -o index.html
# @ colordiff -dw Refs/plan.ref index.html
#
#testfiletree:
# @ $(PYFILETREE) -o index.html
# @ cd $(TESTFILETREEDIR) && $(PWD)/$(PYFILETREE) -plain -o $(PWD)/out.txt
# @ echo DONE TREE
#
#itmal:
# @ rm -f index.html
# cd .. && Etc/$(F) #> Etc/index.html
# head -n 40 index.html
#
#sync:
# scp index.html si:Itmal_E21/
# ssh si "cd Itmal_E21 && git pull"
#
#pub: all
# @ echo "nothing"
#
#all: #$(JSFILES:.js=.okj)
# @#$(foreach i, $(PHPFILES), php -l $i;)
# @ $(foreach i, $(JSFILES), echo "node -c $i"; node -c $i;)
# @ git commit -a -m update && git push
#
#get:
# wget https://gitlab.au.dk/au204573/brightspacemods/-/raw/master/JS/demo.js
#
#sshfs:
# echo "not implemented yet"
#
#ae:
# sudo tail -f /var/log/apache2/error.log
# F21
DAV = /mnt/Dav/ITMAL
DIR = $(DAV)/Fildeling
TEXDIR=/home/cef/ASE/ITMAL/TeX
EXCLUDEPAT=--exclude='*~' --exclude='.ipynb*' --exclude='__pycache__'
VERBOSE=
tree:
Etc/MkHtmlFileTree.py
pub: clean
@ echo "CP lessons, local.."
@ #cp -v -u $(TEXDIR)/lesson01.pdf L01/lesson01.pdf
@ #cp -v -u $(TEXDIR)/lesson02.pdf L02
@ #cp -v -u $(TEXDIR)/lesson03.pdf L03/
@ #cp -v -u $(TEXDIR)/lesson07.pdf L07/
@ #cp -v -u $(TEXDIR)/lesson08.pdf L08/
@ cp -v -u $(TEXDIR)/lesson09.pdf L09/
@ echo "CP lessons, remote.."
@ cp -v -u -r L?? $(DIR)
@ #echo cp -v -u -r `ls L??| grep -v ".ipynb" | grep -v "__pychache__"` $(DIR)
@ echo "CP libitmal, remote.."
@ cp -v -u -r Etc libitmal $(DIR)
#update:
# @ git status
# @ echo -n "Server itu git pull.." && (ssh itu "cd F20_itmal && git pull") || echo "failed"
# @ echo "ALL OK"
# E21
check:
@ grep itundervisining L??/* -R || echo "OK, no 'itundervisining'.." $(VERBOSE)
@ grep "BB-Cou-UUVA-91831" L??/* -R || echo "OK, no 'BB-Cou-UUVA-91831'.." $(VERBOSE)
@#grep "blackboard.au.dk" L??/* -R
@#grep "p\." L01/*.ipynb -R
hasDAV:
@ cat /proc/mounts | grep $(DAV) >/dev/null || mount $(DAV)
@# cat /proc/mounts | grep $(DAV) >/dev/null || (echo "ERROR: DAV dir $(DAV) not mounted" && false)
diff: hasDAV
diff -dwqr -x '*~' -x '.git*' -x 'Makefile' -x 'Solutions' -x 'Old' -x 'Src' -x 'datasets' . $(DIR) || echo "DIFFS(1)!"
@#diff $(TEXDIR)/lesson01.pdf L01/lesson01.pdf || echo "DIFFS(2)!"
@#diff $(TEXDIR)/lesson02.pdf L02/lesson02.pdf || echo "DIFFS(3)!"
@ echo "OK"
cleanremote: hasDAV
@ find $(DIR) -iname '.ipynb_checkpoints' -exec rm -rf {} \; 2>/dev/null || true
@ find $(DIR) -iname '__pycache__' -exec rm -rf {} \; 2>/dev/null || true
@ find $(DIR) -iname '*~' -exec rm -rf {} \; 2>/dev/null || true
clean: cleanremote
@ find . -iname '.ipynb_checkpoints' -exec rm -rf {} \; || true
@ find . -iname '__pycache__' -exec rm -rf {} \; || true
@ find . -iname '*~' -exec rm -rf {} \; || true
all:
@ echo "Doing nothing, perhaps you want to see Etc/Makefile instead"
# F21
#
#DAV = /mnt/Dav/ITMAL
#DIR = $(DAV)/Fildeling
#TEXDIR=/home/cef/ASE/ITMAL/TeX
#EXCLUDEPAT=--exclude='*~' --exclude='.ipynb*' --exclude='__pycache__'
#VERBOSE=
#
#tree:
# Etc/MkHtmlFileTree.py
#
#pub: clean
# @ echo "CP lessons, local.."
# @ #cp -v -u $(TEXDIR)/lesson01.pdf L01/lesson01.pdf
# @ #cp -v -u $(TEXDIR)/lesson02.pdf L02
# @ #cp -v -u $(TEXDIR)/lesson03.pdf L03/
# @ #cp -v -u $(TEXDIR)/lesson07.pdf L07/
# @ #cp -v -u $(TEXDIR)/lesson08.pdf L08/
# @ cp -v -u $(TEXDIR)/lesson09.pdf L09/