#!/usr/bin/env python3
try: FileNotFoundError
except NameError:
FileNotFoundError = IOError
try:
import fcp
except ImportError:
import sys,os
sys.path.append(os.path.expanduser("~/code/fcp"))
import fcp
from afterward import After
images_after = After("images")
from pprint import pprint
import subprocess as s
if True:
call = s.check_call
else:
def call(a):
print(a)
try: s.check_call(a)
except Exception:
import traceback
traceback.print_exc()
raise
import sys,os
sys.path.append(os.path.dirname(sys.argv[0]))
import fixups
from becomer import become
from setupurllib import myretrieve,URLError
from saveint import SavedInteger
from markdown import markdown
from urllib.parse import quote,unquote
from hashlib import sha256
import base64
from bs4 import BeautifulSoup as BeautifulSoupSucks
def BeautifulSoup(s):
return BeautifulSoupSucks(s,"html.parser")
import subprocess as s
import os
from itertools import count
import time
bp = os.path.join
def debug(f):
def w(*a,**kw):
ret = f(*a,**kw)
print(f.__name__,ret,a,kw)
return ret
return w
def soup_for(e):
while e.parent: e = e.parent
return e
def need_update(target,*sources):
if not os.path.exists(target): return True
t = os.stat(target)
for source in sources:
if os.stat(source).st_mtime > t.st_mtime:
return True
return False
indexVersion = 7
class WhichList(list):
def add(self,rpath,which=0):
self.append((rpath,which))
class Site:
noLinks=False
resizeMedia=False
freenet=False
def __init__(self,index,dest):
self.index = index
self.dest = dest
self.images = WhichList()
if 'patience' in os.environ:
patience = int(os.environ['patience'])
else:
patience = 3600*24*7
finished = 'finished' in os.environ
if not finished:
finished = not os.path.exists("unfinished")
class FreenetSite(Site):
# jSite is retarded about links
noLinks=True,
# don't have huge images plz
resizeMedia=True,
freenet=True
def __init__(self, index, dest):
super().__init__(index, dest)
self.chapters = WhichList()
self.important = WhichList()
self.static = WhichList()
self.thumbs = WhichList()
def insert(self, db):
from fcp import FancyNode
def go():
n = FancyNode()
# no remotes by default, since we let FCP decide what to remoteify
try:
yield n.hello()
except Exception as e:
print(type(e))
raise
print("HELLO")
entries = tuple(db.entries())
regulars = []
remotes = []
needurls = []
def doregular(e,stat=None):
regulars.append(LocalEntry(e.path,e.ctype,e.full,e.size))
needurls.append((e,stat))
index = None
for e in entries:
e.full = bp(db.location,e.path)
if e.path == 'index.html':
index = e
continue
if not os.path.exists(e.full):
print(e.full,"lost")
db.problem(e,"path not found")
continue
stat = os.stat(e.full)
if e.uri is not None:
if stat.st_mtime != e.modified or stat.st_size != e.size:
# it changed!
# don't change modified until we're sure it's ready
doregular(e,stat)
else:
elapsed = time.time() - e.seen
if elapsed > patience * 10:
# it hasn't been checked in a LONG while
# better reinsert it so we stop keeping old editions alive
# and making people wait to fetch them
print(e.path,'being reinserted for speed')
doregular(e,None)
elif elapsed > patience:
# it hasn't been checked in a while
ok = yield n.check(e.uri)
if ok:
print(e.path,'was found!')
remotes.append(RemoteEntry(e.path,e.ctype,e.uri))
else:
print(e.path,'was not found')
doregular(e,None)
else:
# it was checked recently
remotes.append(RemoteEntry(e.path,e.ctype,e.uri))
else:
doregular(e,stat)
assert index is not None, "no index!"
print(len(regulars),'regulars',len(remotes),'remotes')
#pprint((regulars,remotes))
#raise SystemExit
if not regulars:
print('no need to update doof')
else:
regulars.insert(0,LocalEntry(index.path,
index.ctype,
index.full,
index.size))
pprint(('eggulars',regulars))
with open('private.uri','rt') as inp:
privateURI = inp.read().rstrip()
if privateURI.startswith('SSK'):
privateURI = 'U' + privateURI[1:]
base,_ = yield n.putComplex(privateURI + self.name + '/0/',regulars, remotes, 'index.html')
del privateURI
if base.startswith('USK@'):
base = base[len('USK@'):]
base,edition = base.rsplit('/',1)
base = 'SSK@'+base+'-'+edition
for e,stat in needurls:
e.uri = base+'/'+e.path
if stat:
e.size = stat.st_size
e.modified = stat.st_mtime
db.saw(e.id,e.uri,e.size,e.modified)
else:
db.setURI(e.id,e.uri)
go()
sites = {
'plain': Site('..','html'),
'freenet': FreenetSite(''.join((
'USK',
'@',
'7wtLtUFBQrqMAkvLCMzYBwXWbjc3-iuz2YkvvlZeZ6E',',',
'Ygw4SHfPErmyLPCcLsAgFCPtkkMnbRf57oghQ0pbOTw',',',
'AQACAAE','/',
'sites','/',
str(indexVersion),'/')),
'freenet')
}
def discard(d,n):
try: del d[n]
except KeyError: pass
def relsymlink(src,dest):
os.symlink(bp(os.path.relpath(".",dest),src),dest)
def memoize(f):
sentinel = []
v = sentinel
def w():
nonlocal v
if v is sentinel: return v
v = f()
return v
return w
def run(site):
topdir,storyName = os.path.split(os.path.abspath("."))
def D(*a):
a = list(a)
a.reverse()
path = None
while a:
if path is None:
path = a.pop()
else:
path = bp(path,a.pop())
try: os.mkdir(path)
except OSError: pass
return path
print(storyName)
top = ".." # er... template/ needed too.
site.html = D(top,site.dest,storyName)
site.temp = D(top,"temp",site.dest,storyName)
site.work = D(top,"temp",site.dest,storyName,"work")
site.media = D("media")
if site.freenet:
from fcp import pathdb
@pathdb.session(site.temp,site.html)
def _(db):
havedb(site,db,storyName)
else:
havedb(site,None,storyName)
def havedb(site,db,storyName):
if 'sitename' in os.environ:
site.name = os.environ['sitename']
else:
site.name = (db and db.site_name()) or storyName
lastTotal = SavedInteger(bp(site.temp,'lastTotal'),0)
updated = False
def findTemplate():
for place in ('.','..','../template'):
for name in ('chapter.xhtml','template.xhtml'):
try:
with open(bp(place,name)) as inp:
return inp.read()
except IOError: pass
raise IOError("Couldn't find template!")
template = findTemplate()
def fixtime(source,dest):
nonlocal updated
stat = os.stat(source)
# utime has a rounding error, so need to set a bit ahead
# or endless dependency regeneration
os.utime(dest,(stat.st_atime+0.1,stat.st_mtime+0.1))
if updated:
updated = max(updated,stat.st_mtime)
else:
updated = stat.st_mtime
def getsource(i):
return 'Chapter'+str(i)+'.md'
def getdest(i):
if i == 1:
return 'index.html'
else:
return 'Chapter'+str(i)+'.html'
# rsync sucks... so can't copy both static/ and media/ to the same place.
# ...or resized
rsyncs = {}
@images_after
def sync(source,rpath,dest):
if site.noLinks:
derp = rsyncs.get(source)
if derp is None:
derp = []
rsyncs[source] = derp
derp.append(rpath)
else:
try: os.symlink(bp(
os.path.relpath(source,dest),
rpath),
bp(dest,rpath))
except OSError: pass
except FileExistsError: pass
assert os.path.exists(bp(dest,rpath)), (dest,rpath)
@images_after
def process_image(rpath,dest,thumbname=None,scale=None):
for source in (site.media,'static'):
full = bp(source,rpath)
if os.path.exists(full): break
else:
raise RuntimeError("Couldn't find image named",rpath)
@memoize
def typeargs():
if rpath.endswith('.png'):
ret = ['-quality','90','-depth','4','-dither','Riemersma']
if wasGif:
ret[:0] = ("-layers","flatten")
else:
ret = ['-quality','60']
return ret
if thumbname:
site.thumbs.add(thumbname, chapter.i)
thumbname = bp(site.html,thumbname)
if need_update(thumbname,full,chapter.source):
args =['convert',
full,
'-resize',
scale]+typeargs()+[
thumbname]
call(args)
if site.resizeMedia:
try: os.mkdir('resized')
except OSError: pass
resized = bp('resized',rpath)
if need_update(resized,full):
call(['convert',
full]+typeargs()+[
'-resize',
'{}@>'.format(600*800),resized])
if os.stat(resized).st_size < os.stat(source).st_size / 1.5:
source = 'resized'
sync(source,rpath,dest)
def wrangleImage(chapter,img,back):
qname = img.attrs['src']
name = unquote(qname)
def freenetize(a,href):
nonlocal name
if 'title' in img.attrs:
title = img.attrs['title']
elif 'title' in a.attrs:
title = a.attrs['title']
else:
title = name
imgdesc = name+'.html'
a.attrs['href'] = imgdesc
site.images.add(imgdesc,2*chapter.i+1)
if need_update(bp(site.html,imgdesc),chapter.source):
with become(bp(site.html,imgdesc)) as out:
out.write(('''
'''+title+'''
\n''').encode('utf-8'))
if href and '://' in href:
out.write((
'Original Source
\n').encode('utf-8'))
out.write(('Back
').encode('utf-8'))
out.write('\n'.encode('utf-8'))
if img.parent.name == 'a':
# make ourselves a sub-site for this image/link
print('freenet-izing',name)
a = img.parent
href = a.attrs['href']
if (href.startswith('http:') or href.startswith('https:')):
freenetize(a,href)
elif 'thumb' in img.attrs or 'scale' in img.attrs:
# create an internal link to the full size image.
a = soup_for(img).new_tag('a')
img.replace_with(a)
a.append(img)
a.attrs['href'] = img['src']
def adjustThumb(scale):
thumbname = 'thumb-'+name
wasGif = False
if thumbname.endswith('.gif'):
wasGif = True
thumbname = thumbname[:-len('.gif')] + ".png"
img.attrs['src'] = thumbname
process_image(name,site.html,thumbname,scale)
if 'thumb' in img.attrs:
print('making thumbnail for',name)
width = int(img.attrs['thumb'])
# scale it down, then zoom in, for tiny (blurry) thumbnails
width = str(int(width*3/4))
img.attrs['style'] = 'width: {}px'.format(width)
adjustThumb(width+'x'+width)
del img.attrs['thumb']
elif 'scale' in img.attrs:
print('scaling for',name)
adjustThumb(img['scale'])
del img['scale']
site.images.add(name,2*chapter.i+2)
def dumb_blorb(uri,name):
d = sha256()
d.update(uri.encode('utf-8'))
return base64.b64encode(d.digest(),b"-_").decode()+name
if db:
db.execute('''CREATE TABLE IF NOT EXISTS URIs (
id INTEGER PRIMARY KEY,
URI TEXT UNIQUE)''')
db.execute('PRAGMA foreign_keys = ON')
db.execute('''CREATE TABLE IF NOT EXISTS URI_path (
id INTEGER PRIMARY KEY,
uri INTEGER REFERENCES URIs(id) ON DELETE CASCADE ON UPDATE CASCADE,
path TEXT)''')
db.execute('CREATE INDEX IF NOT EXISTS by_URI_path ON URI_path(path)')
def blorb(uri,rpath):
with db.cursor() as c:
c.execute('SELECT id FROM URIs WHERE uri = ?',(uri,))
ident = c.fetchone()
if ident:
uri = ident[0]
else:
c.execute('INSERT INTO URIs (uri) VALUES (?)',(uri,))
uri = c.lastrowid
base,ext = os.path.splitext(rpath)
counter = count(0)
# look up by path, so we don't get duplicates
while True:
c.execute('SELECT id FROM URI_path WHERE path = ? and uri != ?',
(rpath,uri))
if not c.fetchone():
break
# try to find an unduplicated path name...
rpath = base + '.' + str(next(counter)) + ext
c.execute('INSERT INTO URI_path (URI,path) VALUES (?,?)',
(uri,rpath))
return rpath
else:
blorb = dumb_blorb
class Done(Exception): pass
class Chapter:
title = None
doc = None
def __init__(self,i):
self.i = i - 1
self.source = bp('chapters',getsource(i))
if not os.path.exists(self.source):
raise Done
# have to be secret about this, since it's my markdown
self.link = getdest(i)
self.dest = bp(site.html,self.link)
self.workpath = bp(site.work,self.link)
def getdoc(self):
if self.doc is None:
try:
with open(self.workpath) as inp:
self.doc = BeautifulSoup(inp)
except FileNotFoundError:
self.regenerate()
def gettitle(self):
if self.title is not None: return self.title
self.getdoc()
self.title = self.doc.find('title').string
return self.title
def check(self):
if 'recheck' in os.environ or need_update(self.dest,self.source):
self.regenerate()
return True
def getImages(self):
for img in self.doc.find_all('img'):
if 'data-fimfiction-src' in img.attrs:
del img.attrs['data-fimfiction-src']
src = img['src']
if src:
print('found image',src)
name = unquote(src.rsplit('/',1)[-1])
assert(name)
if '://' in src:
name = blorb(src,name)
img.attrs['src'] = name
dest = bp(site.media,name)
if not os.path.exists(dest):
@images_after
def getit():
print('retrieving',src,dest)
with open(dest+'.temp',"wb") as out:
while True:
try: myretrieve(src,out)
except URLError as e:
print(dir(e))
e = e.__cause__
e = e.args[0]
print(e.args)
print(e.errno)
raise SystemExit(23)
else:
break
os.rename(dest+'.temp',dest)
if db:
derp = dumb_blorb(src,name)
if os.path.exists(bp(site.media,derp)):
print("found old copy...",derp,'=>',name)
try: os.symlink(derp,dest)
except OSError: pass
else:
getit()
else:
getit()
process_image(name,site.html)
if site.freenet:
wrangleImage(self, img,self.link)
else:
discard(img.attrs,'thumb')
discard(img.attrs,'scale')
def regenerate(self):
print('regenerating',self.i)
if self.doc: del self.doc
title = None
mk,title = markdown(template,self.source,title)
with open(self.workpath,'w') as out:
out.write(str(mk))
self.getdoc()
fixup = fixups.pre(storyName)
if fixup:
print('found pre-fixup for',storyName)
fixup(self.doc,site.html)
self.getImages()
links = None
head = self.doc.find('head')
body = self.doc.find('body')
for link in body.find_all('link'):
link.remove()
head.append(link)
linkbar = self.doc.find('div',id='page') or self.doc.find('body')
def fix(name,href):
for e in self.doc.find_all(name):
e.name = 'a'
e.attrs['href'] = href
def link(rel,i,title=None):
nonlocal links
if isinstance(i,int):
href = chapters[self.i+i].link
else:
href = i
link = self.doc.new_tag('link')
link['href'] = href
link['rel'] = rel
head.append(link)
link = self.doc.new_tag('a')
link['href'] = href
if title is None:
title = rel.title()
link.append(title)
if links:
links.append(' ')
else:
links = self.doc.new_tag('div')
links['id'] = 'links'
linkbar.append(links)
links.append(link)
fix(rel,href)
if self.i > 0:
link('prev',-1)
if self.i + 1 < len(chapters):
link('next',1)
link('up','synopsis.html','Table of Contents')
author = self.doc.find('author')
if author:
author.name = 'div'
author.attrs['class'] = 'author'
for a in self.doc.find_all('a'):
if 'freenet' in a.attrs:
if site.freenet:
a.attrs['href'] = '/'+a.attrs['freenet']
del a.attrs['freenet']
fixup = fixups.get(storyName)
if fixup:
print('found fixup for',storyName)
fixup(self.doc,site.html)
with become(self.dest) as out:
out.write(str(self.doc).encode('utf-8'))
fixtime(self.source,self.dest)
def syncDerp(container,rpath):
source = bp(container,rpath)
if os.path.exists(source):
sync(container,rpath,site.html)
def syncContainer(container):
try: os.mkdir(container)
except OSError: pass
try: rpaths = os.listdir(container)
except FileNotFoundError: return ()
for rpath in rpaths:
if rpath.endswith('.temp'): continue
syncDerp(container,rpath)
return rpaths
rpaths = syncContainer('static')
if site.freenet:
for rpath in rpaths:
site.static.add(rpath,1)
syncContainer('resized')
syncDerp("../template","styles.css")
if site.freenet:
site.important.add("styles.css",0)
# nehhh but this claims the first part loads, so we shouldn't
# stick it at the end:
site.important.add("activelink.png",0)
syncDerp(os.curdir,"activelink.png")
# with the title at the end though, it'll make the whole thing load
# for anyone who tries to view the synopsis. nice
def flushRsyncs():
if rsyncs:
print(site.dest,len(rsyncs),'pending syncs, processing...')
for place,derps in rsyncs.items():
pid = None
try:
pid = s.Popen(['rsync','--verbose',
"--archive",
"--copy-links",
"--from0",
'--files-from=-',
place+'/',site.html+'/'],
stdin=s.PIPE)
for source in derps:
pid.stdin.write((source).encode('utf-8')+b'\0')
finally:
if pid:
pid.stdin.close()
pid.wait()
flushRsyncs()
chapters = []
for which in count(1):
try: chapters.append(Chapter(which))
except Done: break
if len(chapters) > 1 and not finished:
# don't include the current working chapter
chapters = chapters[:-1]
if len(chapters) < lastTotal:
needregen = {len(chapters)-1} # get rid of Next for nonexistent chapter
elif len(chapters) == lastTotal:
needregen = {}
else:
# add Next for last "final" chapter, then regenerate new chapters
needregen = set(range(int(lastTotal)-1,len(chapters)+1))
lastTotal.set(len(chapters))
for i,chapter in enumerate(chapters):
# check all for updates, regenerating if changed
# unconditionally regenerate chapters that need new Prev/Next though
# but don't regenerate them twice.
# also don't bother checking for changed in new chapters
if not i in needregen:
chapter.check()
else:
chapter.regenerate()
if site.freenet:
which = i
if which >= 1:
# 1 is reserved for synopsis.html
which += 1
site.chapters.add(chapter.link,i)
print("### done regenerating ###")
if updated or 'reindex' in os.environ:
with open('synopsis.template.html') as inp:
doc = BeautifulSoup(inp)
for img in doc.find_all('img'):
if not 'src' in img.attrs: continue
print('synopsis image',img.attrs['src'])
site.images.add(unquote(img.attrs['src']))
toc = doc.find('ol',id='toc')
if not toc:
toc = doc.new_tag('ol')
doc.find('body').append(toc)
for i,chapter in enumerate(chapters):
link = doc.new_tag('a')
link['href'] = getdest(chapter.i+1)
link.append(chapter.gettitle() or "???")
li = doc.new_tag('li')
li.append(link)
toc.append(li)
link = doc.new_tag('a')
link.attrs['href'] = site.index
link.append("Stories Index")
toc.insert_after(link)
print('creating synopsis')
with become(bp(site.html,'synopsis.html')) as out:
out.write(str(doc).encode('utf-8'))
if updated:
os.utime(bp(site.html,'synopsis.html'),(updated,updated))
print('afterwards, get images and stuff')
images_after.commit()
print('rsyncs',rsyncs)
flushRsyncs()
if site.freenet:
site.chapters.add('synopsis.html',1)
def see(rpaths,priority):
for rpath,which in rpaths:
try:
ident,uri,seen,modified,ctype,size = db.lookup(
rpath,
priority,
which)
except FileNotFoundError: pass
# the order that we try to insert stuff... then small to big
# priority, which, size
# important files for every page (pretty much just styles.css)
see(site.important,0)
# original content
see(site.chapters,1)
# stuff manually provided (like, not downloaded automatic)
see(site.static,2)
# previews
see(site.thumbs,3)
# full pictures auto downloaded / converted
see(site.images,4)
if 'insert' in os.environ:
print('inserting')
site.insert(db)
if __name__ == '__main__':
for site in sites.values():
run(site)