#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# gitweb recursive dump via blob_plain
import os, re, sys, time, threading, urllib.request, urllib.parse
AUTH = "andrea:Andre@00"
BASE = "http://10.0.30.224/git/"
OUT  = "/var/www/web/IMG/firstad/src"
REPOS = ["empweb","opd6","opdregapp","trainee","webreg_server","webreg_client",
         "localprogram","cdcweb","outsideweb","scy_public","scy","er","opd"]
seen = set()
lock = threading.Lock()

def get(url):
    req = urllib.request.Request(url)
    req.add_header("Authorization", "Basic " + __import__("base64").b64encode(AUTH.encode()).decode())
    req.add_header("User-Agent", "Mozilla/5.0")
    for _ in range(3):
        try:
            with urllib.request.urlopen(req, timeout=12) as r:
                return r.read().decode("utf-8", "ignore")
        except Exception:
            time.sleep(2)
    return ""

def fetch_dir(repo, d, depth):
    if depth > 8: return
    q = urllib.parse.urlencode({"p": repo, "a": "tree", "hb": "HEAD"})
    if d: q += "&f=" + urllib.parse.quote(d)
    html = get(BASE + "?" + q)
    if not html: return
    for m in set(re.findall(r'a=tree;f=([^;&"]+)', html)):
        if m != d and m not in seen:
            with lock: seen.add(m)
            fetch_dir(repo, m, depth+1)
    for m in set(re.findall(r'a=blob_plain;f=([^;&"]+)', html)):
        rel = repo + "/" + m
        fp  = os.path.join(OUT, rel)
        if os.path.exists(fp) and os.path.getsize(fp) > 0: continue
        d2 = os.path.dirname(fp)
        os.makedirs(d2, exist_ok=True)
        q2 = urllib.parse.urlencode({"p": repo, "a": "blob_plain", "hb": "HEAD", "f": m})
        data = get(BASE + "?" + q2)
        if data:
            with open(fp, "w", encoding="utf-8", errors="ignore") as fh: fh.write(data)
            print("OK", rel, len(data))

def run(repo):
    with lock: seen.add(repo)
    fetch_dir(repo, "", 0)
    print("DONE", repo)

if __name__ == "__main__":
    threads = []
    for r in REPOS:
        t = threading.Thread(target=run, args=(r,))
        t.start(); threads.append(t)
    for t in threads: t.join()
    print("ALLDONE")