diff options
| -rwxr-xr-x | scripts/dl_cleanup.py | 429 |
1 files changed, 230 insertions, 199 deletions
diff --git a/scripts/dl_cleanup.py b/scripts/dl_cleanup.py index 83269e68e5..3c26dc8eb5 100755 --- a/scripts/dl_cleanup.py +++ b/scripts/dl_cleanup.py @@ -19,232 +19,263 @@ opt_dryrun = False def parseVer_1234(match, filepath): - progname = match.group(1) - progversion = (int(match.group(2)) << 64) |\ - (int(match.group(3)) << 48) |\ - (int(match.group(4)) << 32) |\ - (int(match.group(5)) << 16) - return (progname, progversion) + progname = match.group(1) + progversion = ( + (int(match.group(2)) << 64) + | (int(match.group(3)) << 48) + | (int(match.group(4)) << 32) + | (int(match.group(5)) << 16) + ) + return (progname, progversion) + def parseVer_123(match, filepath): - progname = match.group(1) - try: - patchlevel = match.group(5) - except IndexError as e: - patchlevel = None - if patchlevel: - patchlevel = ord(patchlevel[0]) - else: - patchlevel = 0 - progversion = (int(match.group(2)) << 64) |\ - (int(match.group(3)) << 48) |\ - (int(match.group(4)) << 32) |\ - patchlevel - return (progname, progversion) + progname = match.group(1) + try: + patchlevel = match.group(5) + except IndexError as e: + patchlevel = None + if patchlevel: + patchlevel = ord(patchlevel[0]) + else: + patchlevel = 0 + progversion = ( + (int(match.group(2)) << 64) + | (int(match.group(3)) << 48) + | (int(match.group(4)) << 32) + | patchlevel + ) + return (progname, progversion) + def parseVer_12(match, filepath): - progname = match.group(1) - try: - patchlevel = match.group(4) - except IndexError as e: - patchlevel = None - if patchlevel: - patchlevel = ord(patchlevel[0]) - else: - patchlevel = 0 - progversion = (int(match.group(2)) << 64) |\ - (int(match.group(3)) << 48) |\ - patchlevel - return (progname, progversion) + progname = match.group(1) + try: + patchlevel = match.group(4) + except IndexError as e: + patchlevel = None + if patchlevel: + patchlevel = ord(patchlevel[0]) + else: + patchlevel = 0 + progversion = (int(match.group(2)) << 64) | (int(match.group(3)) << 48) | patchlevel + return (progname, progversion) + def parseVer_r(match, filepath): - progname = match.group(1) - progversion = (int(match.group(2)) << 64) - return (progname, progversion) + progname = match.group(1) + progversion = int(match.group(2)) << 64 + return (progname, progversion) + def parseVer_ymd_GIT_SHASUM(match, filepath): - progname = match.group(1) - progversion = (int(match.group(2)) << 64) |\ - (int(match.group(3)) << 48) |\ - (int(match.group(4)) << 32) - return (progname, progversion) + progname = match.group(1) + progversion = ( + (int(match.group(2)) << 64) + | (int(match.group(3)) << 48) + | (int(match.group(4)) << 32) + ) + return (progname, progversion) + def parseVer_ymd(match, filepath): - progname = match.group(1) - progversion = (int(match.group(2)) << 64) |\ - (int(match.group(3)) << 48) |\ - (int(match.group(4)) << 32) - return (progname, progversion) + progname = match.group(1) + progversion = ( + (int(match.group(2)) << 64) + | (int(match.group(3)) << 48) + | (int(match.group(4)) << 32) + ) + return (progname, progversion) + def parseVer_GIT(match, filepath): - progname = match.group(1) - st = os.stat(filepath) - progversion = int(st.st_mtime) << 64 - return (progname, progversion) + progname = match.group(1) + st = os.stat(filepath) + progversion = int(st.st_mtime) << 64 + return (progname, progversion) + extensions = ( - ".tar.gz", - ".tar.bz2", - ".tar.xz", - ".orig.tar.gz", - ".orig.tar.bz2", - ".orig.tar.xz", - ".zip", - ".tgz", - ".tbz", - ".txz", + ".tar.gz", + ".tar.bz2", + ".tar.xz", + ".orig.tar.gz", + ".orig.tar.bz2", + ".orig.tar.xz", + ".zip", + ".tgz", + ".tbz", + ".txz", ) versionRegex = ( - (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)\.(\d+)"), parseVer_1234), # xxx-1.2.3.4 - (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)-"), parseVer_ymd_GIT_SHASUM), # xxx-YYYY-MM-DD-GIT_SHASUM - (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)"), parseVer_ymd), # xxx-YYYY-MM-DD - (re.compile(r"(.+)[-_]([0-9a-fA-F]{40,40})"), parseVer_GIT), # xxx-GIT_SHASUM - (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-1.2.3a - (re.compile(r"(.+)[-_](\d+)_(\d+)_(\d+)"), parseVer_123), # xxx-1_2_3 - (re.compile(r"(.+)[-_](\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-1.2a - (re.compile(r"(.+)[-_]r?(\d+)"), parseVer_r), # xxx-r1111 + (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)\.(\d+)"), parseVer_1234), # xxx-1.2.3.4 + ( + re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)-"), + parseVer_ymd_GIT_SHASUM, + ), # xxx-YYYY-MM-DD-GIT_SHASUM + (re.compile(r"(.+)[-_](\d\d\d\d)-?(\d\d)-?(\d\d)"), parseVer_ymd), # xxx-YYYY-MM-DD + (re.compile(r"(.+)[-_]([0-9a-fA-F]{40,40})"), parseVer_GIT), # xxx-GIT_SHASUM + (re.compile(r"(.+)[-_](\d+)\.(\d+)\.(\d+)(\w?)"), parseVer_123), # xxx-1.2.3a + (re.compile(r"(.+)[-_](\d+)_(\d+)_(\d+)"), parseVer_123), # xxx-1_2_3 + (re.compile(r"(.+)[-_](\d+)\.(\d+)(\w?)"), parseVer_12), # xxx-1.2a + (re.compile(r"(.+)[-_]r?(\d+)"), parseVer_r), # xxx-r1111 ) blacklist = [ - ("linux", re.compile(r"linux-\d.*")), - ("gcc", re.compile(r"gcc-.*")), - ("wl_apsta", re.compile(r"wl_apsta.*")), - (".fw", re.compile(r".*\.fw")), - (".arm", re.compile(r".*\.arm")), - (".bin", re.compile(r".*\.bin")), - ("rt-firmware", re.compile(r"RT[\d\w]+_Firmware.*")), + ("linux", re.compile(r"linux-\d.*")), + ("gcc", re.compile(r"gcc-.*")), + ("wl_apsta", re.compile(r"wl_apsta.*")), + (".fw", re.compile(r".*\.fw")), + (".arm", re.compile(r".*\.arm")), + (".bin", re.compile(r".*\.bin")), + ("rt-firmware", re.compile(r"RT[\d\w]+_Firmware.*")), ] -class EntryParseError(Exception): pass + +class EntryParseError(Exception): + pass + class Entry: - def __init__(self, directory, filename): - self.directory = directory - self.filename = filename - self.progname = "" - self.fileext = "" - - for ext in extensions: - if filename.endswith(ext): - filename = filename[0:0-len(ext)] - self.fileext = ext - break - else: - print(self.filename, "has an unknown file-extension") - raise EntryParseError("ext") - for (regex, parseVersion) in versionRegex: - match = regex.match(filename) - if match: - (self.progname, self.version) = parseVersion( - match, directory + "/" + filename + self.fileext) - break - else: - print(self.filename, "has an unknown version pattern") - raise EntryParseError("ver") - - def getPath(self): - return (self.directory + "/" + self.filename).replace("//", "/") - - def deleteFile(self): - path = self.getPath() - print("Deleting", path) - if not opt_dryrun: - os.unlink(path) - - def __ge__(self, y): - return self.version >= y.version + def __init__(self, directory, filename): + self.directory = directory + self.filename = filename + self.progname = "" + self.fileext = "" + + for ext in extensions: + if filename.endswith(ext): + filename = filename[0 : 0 - len(ext)] + self.fileext = ext + break + else: + print(self.filename, "has an unknown file-extension") + raise EntryParseError("ext") + for (regex, parseVersion) in versionRegex: + match = regex.match(filename) + if match: + (self.progname, self.version) = parseVersion( + match, directory + "/" + filename + self.fileext + ) + break + else: + print(self.filename, "has an unknown version pattern") + raise EntryParseError("ver") + + def getPath(self): + return (self.directory + "/" + self.filename).replace("//", "/") + + def deleteFile(self): + path = self.getPath() + print("Deleting", path) + if not opt_dryrun: + os.unlink(path) + + def __ge__(self, y): + return self.version >= y.version + def usage(): - print("OpenWrt download directory cleanup utility") - print("Usage: " + sys.argv[0] + " [OPTIONS] <path/to/dl>") - print("") - print(" -d|--dry-run Do a dry-run. Don't delete any files") - print(" -B|--show-blacklist Show the blacklist and exit") - print(" -w|--whitelist ITEM Remove ITEM from blacklist") + print("OpenWrt download directory cleanup utility") + print("Usage: " + sys.argv[0] + " [OPTIONS] <path/to/dl>") + print("") + print(" -d|--dry-run Do a dry-run. Don't delete any files") + print(" -B|--show-blacklist Show the blacklist and exit") + print(" -w|--whitelist ITEM Remove ITEM from blacklist") + def main(argv): - global opt_dryrun - - try: - (opts, args) = getopt.getopt(argv[1:], - "hdBw:", - [ "help", "dry-run", "show-blacklist", "whitelist=", ]) - if len(args) != 1: - usage() - return 1 - except getopt.GetoptError as e: - usage() - return 1 - directory = args[0] - - if not os.path.exists(directory): - print("Can't find dl path", directory) - return 1 - - for (o, v) in opts: - if o in ("-h", "--help"): - usage() - return 0 - if o in ("-d", "--dry-run"): - opt_dryrun = True - if o in ("-w", "--whitelist"): - for i in range(0, len(blacklist)): - (name, regex) = blacklist[i] - if name == v: - del blacklist[i] - break - else: - print("Whitelist error: Item", v,\ - "is not in blacklist") - return 1 - if o in ("-B", "--show-blacklist"): - for (name, regex) in blacklist: - sep = "\t\t" - if len(name) >= 8: - sep = "\t" - print("%s%s(%s)" % (name, sep, regex.pattern)) - return 0 - - # Create a directory listing and parse the file names. - entries = [] - for filename in os.listdir(directory): - if filename == "." or filename == "..": - continue - for (name, regex) in blacklist: - if regex.match(filename): - if opt_dryrun: - print(filename, "is blacklisted") - break - else: - try: - entries.append(Entry(directory, filename)) - except EntryParseError as e: - pass - - # Create a map of programs - progmap = {} - for entry in entries: - if entry.progname in progmap.keys(): - progmap[entry.progname].append(entry) - else: - progmap[entry.progname] = [entry,] - - # Traverse the program map and delete everything but the last version - for prog in progmap: - lastVersion = None - versions = progmap[prog] - for version in versions: - if lastVersion is None or version >= lastVersion: - lastVersion = version - if lastVersion: - for version in versions: - if version is not lastVersion: - version.deleteFile() - if opt_dryrun: - print("Keeping", lastVersion.getPath()) - - return 0 + global opt_dryrun + + try: + (opts, args) = getopt.getopt( + argv[1:], + "hdBw:", + [ + "help", + "dry-run", + "show-blacklist", + "whitelist=", + ], + ) + if len(args) != 1: + usage() + return 1 + except getopt.GetoptError as e: + usage() + return 1 + directory = args[0] + + if not os.path.exists(directory): + print("Can't find dl path", directory) + return 1 + + for (o, v) in opts: + if o in ("-h", "--help"): + usage() + return 0 + if o in ("-d", "--dry-run"): + opt_dryrun = True + if o in ("-w", "--whitelist"): + for i in range(0, len(blacklist)): + (name, regex) = blacklist[i] + if name == v: + del blacklist[i] + break + else: + print("Whitelist error: Item", v, "is not in blacklist") + return 1 + if o in ("-B", "--show-blacklist"): + for (name, regex) in blacklist: + sep = "\t\t" + if len(name) >= 8: + sep = "\t" + print("%s%s(%s)" % (name, sep, regex.pattern)) + return 0 + + # Create a directory listing and parse the file names. + entries = [] + for filename in os.listdir(directory): + if filename == "." or filename == "..": + continue + for (name, regex) in blacklist: + if regex.match(filename): + if opt_dryrun: + print(filename, "is blacklisted") + break + else: + try: + entries.append(Entry(directory, filename)) + except EntryParseError as e: + pass + + # Create a map of programs + progmap = {} + for entry in entries: + if entry.progname in progmap.keys(): + progmap[entry.progname].append(entry) + else: + progmap[entry.progname] = [ + entry, + ] + + # Traverse the program map and delete everything but the last version + for prog in progmap: + lastVersion = None + versions = progmap[prog] + for version in versions: + if lastVersion is None or version >= lastVersion: + lastVersion = version + if lastVersion: + for version in versions: + if version is not lastVersion: + version.deleteFile() + if opt_dryrun: + print("Keeping", lastVersion.getPath()) + + return 0 + if __name__ == "__main__": - sys.exit(main(sys.argv)) + sys.exit(main(sys.argv)) |