From 2a3272b7d0ac83fa9f3d3d73f0bfc127bd8d1949 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Fri, 10 Jun 2022 20:38:56 +0200 Subject: [PATCH 01/20] patch radiolib on-the-fly while building. --- bin/apply_patches.py | 27 + bin/patch_ng.py | 1381 +++++++++++++++++ .../0001-RadioLib-SPItransfer-virtual.patch | 12 + platformio.ini | 7 + 4 files changed, 1427 insertions(+) create mode 100644 bin/apply_patches.py create mode 100644 bin/patch_ng.py create mode 100644 patches/0001-RadioLib-SPItransfer-virtual.patch diff --git a/bin/apply_patches.py b/bin/apply_patches.py new file mode 100644 index 000000000..d8c183b05 --- /dev/null +++ b/bin/apply_patches.py @@ -0,0 +1,27 @@ +from os.path import join, isfile + +Import("env") + +LIBRARY_DIR = join (env["PROJECT_LIBDEPS_DIR"], env["PIOENV"], "RadioLib") +patchflag_path = join(LIBRARY_DIR, ".patching-done") +patch = join(env["PROJECT_DIR"], "bin", "patch_ng.py") + +# patch file only if we didn't do it before +if not isfile(join(LIBRARY_DIR, ".patching-done")): + original_path = join(LIBRARY_DIR) + patch_file = join(env["PROJECT_DIR"], "patches", "0001-RadioLib-SPItransfer-virtual.patch") + + assert isfile(patch_file) + + env.Execute( + env.VerboseAction( + "$PYTHONEXE %s -p 1 --directory=%s %s" % (patch, original_path, patch_file) + , "Applying patch to RadioLib" + ) + ) + + def _touch(path): + with open(path, "w") as fp: + fp.write("") + + env.Execute(lambda *args, **kwargs: _touch(patchflag_path)) \ No newline at end of file diff --git a/bin/patch_ng.py b/bin/patch_ng.py new file mode 100644 index 000000000..088fbbab3 --- /dev/null +++ b/bin/patch_ng.py @@ -0,0 +1,1381 @@ +#!/usr/bin/env python +""" + Patch utility to apply unified diffs + + Brute-force line-by-line non-recursive parsing + + Copyright (c) 2008-2016 anatoly techtonik + Available under the terms of MIT license + +--- + The MIT License (MIT) + + Copyright (c) 2019 JFrog LTD + + Permission is hereby granted, free of charge, to any person obtaining a copy of this software + and associated documentation files (the "Software"), to deal in the Software without + restriction, including without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all copies or + substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, + INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +""" +from __future__ import print_function + +__author__ = "Conan.io " +__version__ = "1.17.4" +__license__ = "MIT" +__url__ = "https://github.com/conan-io/python-patch" + +import copy +import logging +import re +import tempfile +import codecs + +# cStringIO doesn't support unicode in 2.5 +try: + from StringIO import StringIO +except ImportError: + from io import BytesIO as StringIO # python 3 +try: + import urllib2 as urllib_request +except ImportError: + import urllib.request as urllib_request + +from os.path import exists, isfile, abspath +import os +import posixpath +import shutil +import sys +import stat + + +PY3K = sys.version_info >= (3, 0) + +# PEP 3114 +if not PY3K: + compat_next = lambda gen: gen.next() +else: + compat_next = lambda gen: gen.__next__() + +def tostr(b): + """ Python 3 bytes encoder. Used to print filename in + diffstat output. Assumes that filenames are in utf-8. + """ + if not PY3K: + return b + + # [ ] figure out how to print non-utf-8 filenames without + # information loss + return b.decode('utf-8') + + +#------------------------------------------------ +# Logging is controlled by logger named after the +# module name (e.g. 'patch' for patch_ng.py module) + +logger = logging.getLogger("patch_ng") + +debug = logger.debug +info = logger.info +warning = logger.warning +error = logger.error + +class NullHandler(logging.Handler): + """ Copied from Python 2.7 to avoid getting + `No handlers could be found for logger "patch"` + http://bugs.python.org/issue16539 + """ + def handle(self, record): + pass + def emit(self, record): + pass + def createLock(self): + self.lock = None + +streamhandler = logging.StreamHandler() + +# initialize logger itself +logger.addHandler(NullHandler()) + +debugmode = False + +def setdebug(): + global debugmode, streamhandler + + debugmode = True + loglevel = logging.DEBUG + logformat = "%(levelname)8s %(message)s" + logger.setLevel(loglevel) + + if streamhandler not in logger.handlers: + # when used as a library, streamhandler is not added + # by default + logger.addHandler(streamhandler) + + streamhandler.setFormatter(logging.Formatter(logformat)) + + +#------------------------------------------------ +# Constants for Patch/PatchSet types + +DIFF = PLAIN = "plain" +GIT = "git" +HG = MERCURIAL = "mercurial" +SVN = SUBVERSION = "svn" +# mixed type is only actual when PatchSet contains +# Patches of different type +MIXED = MIXED = "mixed" + + +#------------------------------------------------ +# Helpers (these could come with Python stdlib) + +# x...() function are used to work with paths in +# cross-platform manner - all paths use forward +# slashes even on Windows. + +def xisabs(filename): + """ Cross-platform version of `os.path.isabs()` + Returns True if `filename` is absolute on + Linux, OS X or Windows. + """ + if filename.startswith(b'/'): # Linux/Unix + return True + elif filename.startswith(b'\\'): # Windows + return True + elif re.match(b'\\w:[\\\\/]', filename): # Windows + return True + return False + +def xnormpath(path): + """ Cross-platform version of os.path.normpath """ + # replace escapes and Windows slashes + normalized = posixpath.normpath(path).replace(b'\\', b'/') + # fold the result + return posixpath.normpath(normalized) + +def xstrip(filename): + """ Make relative path out of absolute by stripping + prefixes used on Linux, OS X and Windows. + + This function is critical for security. + """ + while xisabs(filename): + # strip windows drive with all slashes + if re.match(b'\\w:[\\\\/]', filename): + filename = re.sub(b'^\\w+:[\\\\/]+', b'', filename) + # strip all slashes + elif re.match(b'[\\\\/]', filename): + filename = re.sub(b'^[\\\\/]+', b'', filename) + return filename + + +def safe_unlink(filepath): + os.chmod(filepath, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) + os.unlink(filepath) + + +#----------------------------------------------- +# Main API functions + +def fromfile(filename): + """ Parse patch file. If successful, returns + PatchSet() object. Otherwise returns False. + """ + patchset = PatchSet() + debug("reading %s" % filename) + fp = open(filename, "rb") + res = patchset.parse(fp) + fp.close() + if res == True: + return patchset + return False + + +def fromstring(s): + """ Parse text string and return PatchSet() + object (or False if parsing fails) + """ + ps = PatchSet( StringIO(s) ) + if ps.errors == 0: + return ps + return False + + +def fromurl(url): + """ Parse patch from an URL, return False + if an error occured. Note that this also + can throw urlopen() exceptions. + """ + ps = PatchSet( urllib_request.urlopen(url) ) + if ps.errors == 0: + return ps + return False + + +# --- Utility functions --- +# [ ] reuse more universal pathsplit() +def pathstrip(path, n): + """ Strip n leading components from the given path """ + pathlist = [path] + while os.path.dirname(pathlist[0]) != b'': + pathlist[0:1] = os.path.split(pathlist[0]) + return b'/'.join(pathlist[n:]) +# --- /Utility function --- + + +def decode_text(text): + encodings = {codecs.BOM_UTF8: "utf_8_sig", + codecs.BOM_UTF16_BE: "utf_16_be", + codecs.BOM_UTF16_LE: "utf_16_le", + codecs.BOM_UTF32_BE: "utf_32_be", + codecs.BOM_UTF32_LE: "utf_32_le", + b'\x2b\x2f\x76\x38': "utf_7", + b'\x2b\x2f\x76\x39': "utf_7", + b'\x2b\x2f\x76\x2b': "utf_7", + b'\x2b\x2f\x76\x2f': "utf_7", + b'\x2b\x2f\x76\x38\x2d': "utf_7"} + for bom in sorted(encodings, key=len, reverse=True): + if text.startswith(bom): + try: + return text[len(bom):].decode(encodings[bom]) + except UnicodeDecodeError: + continue + decoders = ["utf-8", "Windows-1252"] + for decoder in decoders: + try: + return text.decode(decoder) + except UnicodeDecodeError: + continue + logger.warning("can't decode %s" % str(text)) + return text.decode("utf-8", "ignore") # Ignore not compatible characters + + +def to_file_bytes(content): + if PY3K: + if not isinstance(content, bytes): + content = bytes(content, "utf-8") + elif isinstance(content, unicode): + content = content.encode("utf-8") + return content + + +def load(path, binary=False): + """ Loads a file content """ + with open(path, 'rb') as handle: + tmp = handle.read() + return tmp if binary else decode_text(tmp) + + +def save(path, content, only_if_modified=False): + """ + Saves a file with given content + Params: + path: path to write file to + content: contents to save in the file + only_if_modified: file won't be modified if the content hasn't changed + """ + try: + os.makedirs(os.path.dirname(path)) + except Exception: + pass + + new_content = to_file_bytes(content) + + if only_if_modified and os.path.exists(path): + old_content = load(path, binary=True) + if old_content == new_content: + return + + with open(path, "wb") as handle: + handle.write(new_content) + + +class Hunk(object): + """ Parsed hunk data container (hunk starts with @@ -R +R @@) """ + + def __init__(self): + self.startsrc=None #: line count starts with 1 + self.linessrc=None + self.starttgt=None + self.linestgt=None + self.invalid=False + self.desc='' + self.text=[] + + +class Patch(object): + """ Patch for a single file. + If used as an iterable, returns hunks. + """ + def __init__(self): + self.source = None + self.target = None + self.hunks = [] + self.hunkends = [] + self.header = [] + + self.type = None + + def __iter__(self): + for h in self.hunks: + yield h + + +class PatchSet(object): + """ PatchSet is a patch parser and container. + When used as an iterable, returns patches. + """ + + def __init__(self, stream=None): + # --- API accessible fields --- + + # name of the PatchSet (filename or ...) + self.name = None + # patch set type - one of constants + self.type = None + + # list of Patch objects + self.items = [] + + self.errors = 0 # fatal parsing errors + self.warnings = 0 # non-critical warnings + # --- /API --- + + if stream: + self.parse(stream) + + def __len__(self): + return len(self.items) + + def __iter__(self): + for i in self.items: + yield i + + def parse(self, stream): + """ parse unified diff + return True on success + """ + lineends = dict(lf=0, crlf=0, cr=0) + nexthunkno = 0 #: even if index starts with 0 user messages number hunks from 1 + + p = None + hunk = None + # hunkactual variable is used to calculate hunk lines for comparison + hunkactual = dict(linessrc=None, linestgt=None) + + + class wrapumerate(enumerate): + """Enumerate wrapper that uses boolean end of stream status instead of + StopIteration exception, and properties to access line information. + """ + + def __init__(self, *args, **kwargs): + # we don't call parent, it is magically created by __new__ method + + self._exhausted = False + self._lineno = False # after end of stream equal to the num of lines + self._line = False # will be reset to False after end of stream + + def next(self): + """Try to read the next line and return True if it is available, + False if end of stream is reached.""" + if self._exhausted: + return False + + try: + self._lineno, self._line = compat_next(super(wrapumerate, self)) + except StopIteration: + self._exhausted = True + self._line = False + return False + return True + + @property + def is_empty(self): + return self._exhausted + + @property + def line(self): + return self._line + + @property + def lineno(self): + return self._lineno + + # define states (possible file regions) that direct parse flow + headscan = True # start with scanning header + filenames = False # lines starting with --- and +++ + + hunkhead = False # @@ -R +R @@ sequence + hunkbody = False # + hunkskip = False # skipping invalid hunk mode + + hunkparsed = False # state after successfully parsed hunk + + # regexp to match start of hunk, used groups - 1,3,4,6 + re_hunk_start = re.compile(b"^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))? @@") + + self.errors = 0 + # temp buffers for header and filenames info + header = [] + srcname = None + tgtname = None + + # start of main cycle + # each parsing block already has line available in fe.line + fe = wrapumerate(stream) + while fe.next(): + + # -- deciders: these only switch state to decide who should process + # -- line fetched at the start of this cycle + if hunkparsed: + hunkparsed = False + if re_hunk_start.match(fe.line): + hunkhead = True + elif fe.line.startswith(b"--- "): + filenames = True + else: + headscan = True + # -- ------------------------------------ + + # read out header + if headscan: + while not fe.is_empty and not fe.line.startswith(b"--- "): + header.append(fe.line) + fe.next() + if fe.is_empty: + if p is None: + debug("no patch data found") # error is shown later + self.errors += 1 + else: + info("%d unparsed bytes left at the end of stream" % len(b''.join(header))) + self.warnings += 1 + # TODO check for \No new line at the end.. + # TODO test for unparsed bytes + # otherwise error += 1 + # this is actually a loop exit + continue + + headscan = False + # switch to filenames state + filenames = True + + line = fe.line + lineno = fe.lineno + + + # hunkskip and hunkbody code skipped until definition of hunkhead is parsed + if hunkbody: + # [x] treat empty lines inside hunks as containing single space + # (this happens when diff is saved by copy/pasting to editor + # that strips trailing whitespace) + if line.strip(b"\r\n") == b"": + debug("expanding empty line in a middle of hunk body") + self.warnings += 1 + line = b' ' + line + + # process line first + if re.match(b"^[- \\+\\\\]", line): + # gather stats about line endings + if line.endswith(b"\r\n"): + p.hunkends["crlf"] += 1 + elif line.endswith(b"\n"): + p.hunkends["lf"] += 1 + elif line.endswith(b"\r"): + p.hunkends["cr"] += 1 + + if line.startswith(b"-"): + hunkactual["linessrc"] += 1 + elif line.startswith(b"+"): + hunkactual["linestgt"] += 1 + elif not line.startswith(b"\\"): + hunkactual["linessrc"] += 1 + hunkactual["linestgt"] += 1 + hunk.text.append(line) + # todo: handle \ No newline cases + else: + warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + + # check exit conditions + if hunkactual["linessrc"] > hunk.linessrc or hunkactual["linestgt"] > hunk.linestgt: + warning("extra lines for hunk no.%d at %d for target %s" % (nexthunkno, lineno+1, p.target)) + # add hunk status node + hunk.invalid = True + p.hunks.append(hunk) + self.errors += 1 + # switch to hunkskip state + hunkbody = False + hunkskip = True + elif hunk.linessrc == hunkactual["linessrc"] and hunk.linestgt == hunkactual["linestgt"]: + # hunk parsed successfully + p.hunks.append(hunk) + # switch to hunkparsed state + hunkbody = False + hunkparsed = True + + # detect mixed window/unix line ends + ends = p.hunkends + if ((ends["cr"]!=0) + (ends["crlf"]!=0) + (ends["lf"]!=0)) > 1: + warning("inconsistent line ends in patch hunks for %s" % p.source) + self.warnings += 1 + if debugmode: + debuglines = dict(ends) + debuglines.update(file=p.target, hunk=nexthunkno) + debug("crlf: %(crlf)d lf: %(lf)d cr: %(cr)d\t - file: %(file)s hunk: %(hunk)d" % debuglines) + # fetch next line + continue + + if hunkskip: + if re_hunk_start.match(line): + # switch to hunkhead state + hunkskip = False + hunkhead = True + elif line.startswith(b"--- "): + # switch to filenames state + hunkskip = False + filenames = True + if debugmode and len(self.items) > 0: + debug("- %2d hunks for %s" % (len(p.hunks), p.source)) + + if filenames: + if line.startswith(b"--- "): + if srcname != None: + # XXX testcase + warning("skipping false patch for %s" % srcname) + srcname = None + # XXX header += srcname + # double source filename line is encountered + # attempt to restart from this second line + + # Files dated at Unix epoch don't exist, e.g.: + # '1970-01-01 01:00:00.000000000 +0100' + # They include timezone offsets. + # .. which can be parsed (if we remove the nanoseconds) + # .. by strptime() with: + # '%Y-%m-%d %H:%M:%S %z' + # .. but unfortunately this relies on the OSes libc + # strptime function and %z support is patchy, so we drop + # everything from the . onwards and group the year and time + # separately. + re_filename_date_time = b"^--- ([^\t]+)(?:\s([0-9-]+)\s([0-9:]+)|.*)" + match = re.match(re_filename_date_time, line) + # todo: support spaces in filenames + if match: + srcname = match.group(1).strip() + date = match.group(2) + time = match.group(3) + if (date == b'1970-01-01' or date == b'1969-12-31') and time.split(b':',1)[1] == b'00:00': + srcname = b'/dev/null' + else: + warning("skipping invalid filename at line %d" % (lineno+1)) + self.errors += 1 + # XXX p.header += line + # switch back to headscan state + filenames = False + headscan = True + elif not line.startswith(b"+++ "): + if srcname != None: + warning("skipping invalid patch with no target for %s" % srcname) + self.errors += 1 + srcname = None + # XXX header += srcname + # XXX header += line + else: + # this should be unreachable + warning("skipping invalid target patch") + filenames = False + headscan = True + else: + if tgtname != None: + # XXX seems to be a dead branch + warning("skipping invalid patch - double target at line %d" % (lineno+1)) + self.errors += 1 + srcname = None + tgtname = None + # XXX header += srcname + # XXX header += tgtname + # XXX header += line + # double target filename line is encountered + # switch back to headscan state + filenames = False + headscan = True + else: + re_filename_date_time = b"^\+\+\+ ([^\t]+)(?:\s([0-9-]+)\s([0-9:]+)|.*)" + match = re.match(re_filename_date_time, line) + if not match: + warning("skipping invalid patch - no target filename at line %d" % (lineno+1)) + self.errors += 1 + srcname = None + # switch back to headscan state + filenames = False + headscan = True + else: + tgtname = match.group(1).strip() + date = match.group(2) + time = match.group(3) + if (date == b'1970-01-01' or date == b'1969-12-31') and time.split(b':',1)[1] == b'00:00': + tgtname = b'/dev/null' + if p: # for the first run p is None + self.items.append(p) + p = Patch() + p.source = srcname + srcname = None + p.target = tgtname + tgtname = None + p.header = header + header = [] + # switch to hunkhead state + filenames = False + hunkhead = True + nexthunkno = 0 + p.hunkends = lineends.copy() + continue + + if hunkhead: + match = re.match(b"^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))? @@(.*)", line) + if not match: + if not p.hunks: + warning("skipping invalid patch with no hunks for file %s" % p.source) + self.errors += 1 + # XXX review switch + # switch to headscan state + hunkhead = False + headscan = True + continue + else: + # TODO review condition case + # switch to headscan state + hunkhead = False + headscan = True + else: + hunk = Hunk() + hunk.startsrc = int(match.group(1)) + hunk.linessrc = 1 + if match.group(3): hunk.linessrc = int(match.group(3)) + hunk.starttgt = int(match.group(4)) + hunk.linestgt = 1 + if match.group(6): hunk.linestgt = int(match.group(6)) + hunk.invalid = False + hunk.desc = match.group(7)[1:].rstrip() + hunk.text = [] + + hunkactual["linessrc"] = hunkactual["linestgt"] = 0 + + # switch to hunkbody state + hunkhead = False + hunkbody = True + nexthunkno += 1 + continue + + # /while fe.next() + + if p: + self.items.append(p) + + if not hunkparsed: + if hunkskip: + warning("warning: finished with errors, some hunks may be invalid") + elif headscan: + if len(self.items) == 0: + warning("error: no patch data found!") + return False + else: # extra data at the end of file + pass + else: + warning("error: patch stream is incomplete!") + self.errors += 1 + if len(self.items) == 0: + return False + + if debugmode and len(self.items) > 0: + debug("- %2d hunks for %s" % (len(p.hunks), p.source)) + + # XXX fix total hunks calculation + debug("total files: %d total hunks: %d" % (len(self.items), + sum(len(p.hunks) for p in self.items))) + + # ---- detect patch and patchset types ---- + for idx, p in enumerate(self.items): + self.items[idx].type = self._detect_type(p) + + types = set([p.type for p in self.items]) + if len(types) > 1: + self.type = MIXED + else: + self.type = types.pop() + # -------- + + self._normalize_filenames() + + return (self.errors == 0) + + def _detect_type(self, p): + """ detect and return type for the specified Patch object + analyzes header and filenames info + + NOTE: must be run before filenames are normalized + """ + + # check for SVN + # - header starts with Index: + # - next line is ===... delimiter + # - filename is followed by revision number + # TODO add SVN revision + if (len(p.header) > 1 and p.header[-2].startswith(b"Index: ") + and p.header[-1].startswith(b"="*67)): + return SVN + + # common checks for both HG and GIT + DVCS = ((p.source.startswith(b'a/') or p.source == b'/dev/null') + and (p.target.startswith(b'b/') or p.target == b'/dev/null')) + + # GIT type check + # - header[-2] is like "diff --git a/oldname b/newname" + # - header[-1] is like "index .. " + # TODO add git rename diffs and add/remove diffs + # add git diff with spaced filename + # TODO http://www.kernel.org/pub/software/scm/git/docs/git-diff.html + + # Git patch header len is 2 min + if len(p.header) > 1: + # detect the start of diff header - there might be some comments before + for idx in reversed(range(len(p.header))): + if p.header[idx].startswith(b"diff --git"): + break + if p.header[idx].startswith(b'diff --git a/'): + if (idx+1 < len(p.header) + and re.match(b'(?:index \\w{7}..\\w{7} \\d{6}|new file mode \\d*)', p.header[idx+1])): + if DVCS: + return GIT + + # HG check + # + # - for plain HG format header is like "diff -r b2d9961ff1f5 filename" + # - for Git-style HG patches it is "diff --git a/oldname b/newname" + # - filename starts with a/, b/ or is equal to /dev/null + # - exported changesets also contain the header + # # HG changeset patch + # # User name@example.com + # ... + # TODO add MQ + # TODO add revision info + if len(p.header) > 0: + if DVCS and re.match(b'diff -r \\w{12} .*', p.header[-1]): + return HG + if DVCS and p.header[-1].startswith(b'diff --git a/'): + if len(p.header) == 1: # native Git patch header len is 2 + return HG + elif p.header[0].startswith(b'# HG changeset patch'): + return HG + + return PLAIN + + + def _normalize_filenames(self): + """ sanitize filenames, normalizing paths, i.e.: + 1. strip a/ and b/ prefixes from GIT and HG style patches + 2. remove all references to parent directories (with warning) + 3. translate any absolute paths to relative (with warning) + + [x] always use forward slashes to be crossplatform + (diff/patch were born as a unix utility after all) + + return None + """ + if debugmode: + debug("normalize filenames") + for i,p in enumerate(self.items): + if debugmode: + debug(" patch type = %s" % p.type) + debug(" source = %s" % p.source) + debug(" target = %s" % p.target) + if p.type in (HG, GIT): + debug("stripping a/ and b/ prefixes") + if p.source != b'/dev/null': + if not p.source.startswith(b"a/"): + warning("invalid source filename") + else: + p.source = p.source[2:] + if p.target != b'/dev/null': + if not p.target.startswith(b"b/"): + warning("invalid target filename") + else: + p.target = p.target[2:] + + p.source = xnormpath(p.source) + p.target = xnormpath(p.target) + + sep = b'/' # sep value can be hardcoded, but it looks nice this way + + # references to parent are not allowed + if p.source.startswith(b".." + sep): + warning("error: stripping parent path for source file patch no.%d" % (i+1)) + self.warnings += 1 + while p.source.startswith(b".." + sep): + p.source = p.source.partition(sep)[2] + if p.target.startswith(b".." + sep): + warning("error: stripping parent path for target file patch no.%d" % (i+1)) + self.warnings += 1 + while p.target.startswith(b".." + sep): + p.target = p.target.partition(sep)[2] + # absolute paths are not allowed + if (xisabs(p.source) and p.source != b'/dev/null') or \ + (xisabs(p.target) and p.target != b'/dev/null'): + warning("error: absolute paths are not allowed - file no.%d" % (i+1)) + self.warnings += 1 + if xisabs(p.source) and p.source != b'/dev/null': + warning("stripping absolute path from source name '%s'" % p.source) + p.source = xstrip(p.source) + if xisabs(p.target) and p.target != b'/dev/null': + warning("stripping absolute path from target name '%s'" % p.target) + p.target = xstrip(p.target) + + self.items[i].source = p.source + self.items[i].target = p.target + + + def diffstat(self): + """ calculate diffstat and return as a string + Notes: + - original diffstat ouputs target filename + - single + or - shouldn't escape histogram + """ + names = [] + insert = [] + delete = [] + delta = 0 # size change in bytes + namelen = 0 + maxdiff = 0 # max number of changes for single file + # (for histogram width calculation) + for patch in self.items: + i,d = 0,0 + for hunk in patch.hunks: + for line in hunk.text: + if line.startswith(b'+'): + i += 1 + delta += len(line)-1 + elif line.startswith(b'-'): + d += 1 + delta -= len(line)-1 + names.append(patch.target) + insert.append(i) + delete.append(d) + namelen = max(namelen, len(patch.target)) + maxdiff = max(maxdiff, i+d) + output = '' + statlen = len(str(maxdiff)) # stats column width + for i,n in enumerate(names): + # %-19s | %-4d %s + format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n" + + hist = '' + # -- calculating histogram -- + width = len(format % ('', '', '')) + histwidth = max(2, 80 - width) + if maxdiff < histwidth: + hist = "+"*insert[i] + "-"*delete[i] + else: + iratio = (float(insert[i]) / maxdiff) * histwidth + dratio = (float(delete[i]) / maxdiff) * histwidth + + # make sure every entry gets at least one + or - + iwidth = 1 if 0 < iratio < 1 else int(iratio) + dwidth = 1 if 0 < dratio < 1 else int(dratio) + #print(iratio, dratio, iwidth, dwidth, histwidth) + hist = "+"*int(iwidth) + "-"*int(dwidth) + # -- /calculating +- histogram -- + output += (format % (tostr(names[i]), str(insert[i] + delete[i]), hist)) + + output += (" %d files changed, %d insertions(+), %d deletions(-), %+d bytes" + % (len(names), sum(insert), sum(delete), delta)) + return output + + + def findfiles(self, old, new): + """ return tuple of source file, target file """ + if old == b'/dev/null': + handle, abspath = tempfile.mkstemp(suffix='pypatch') + abspath = abspath.encode() + # The source file must contain a line for the hunk matching to succeed. + os.write(handle, b' ') + os.close(handle) + if not exists(new): + handle = open(new, 'wb') + handle.close() + return abspath, new + elif exists(old): + return old, old + elif exists(new): + return new, new + elif new == b'/dev/null': + return None, None + else: + # [w] Google Code generates broken patches with its online editor + debug("broken patch from Google Code, stripping prefixes..") + if old.startswith(b'a/') and new.startswith(b'b/'): + old, new = old[2:], new[2:] + debug(" %s" % old) + debug(" %s" % new) + if exists(old): + return old, old + elif exists(new): + return new, new + return None, None + + def _strip_prefix(self, filename): + if filename.startswith(b'a/') or filename.startswith(b'b/'): + return filename[2:] + return filename + + def decode_clean(self, path, prefix): + path = path.decode("utf-8").replace("\\", "/") + if path.startswith(prefix): + path = path[2:] + return path + + def strip_path(self, path, base_path, strip=0): + tokens = path.split("/") + if len(tokens) > 1: + tokens = tokens[strip:] + path = "/".join(tokens) + if base_path: + path = os.path.join(base_path, path) + return path + # account for new and deleted files, upstream dep won't fix them + + + + + def apply(self, strip=0, root=None, fuzz=False): + """ Apply parsed patch, optionally stripping leading components + from file paths. `root` parameter specifies working dir. + :param strip: Strip patch path + :param root: Folder to apply the patch + :param fuzz: Accept fuzzy patches + return True on success + """ + items = [] + for item in self.items: + source = self.decode_clean(item.source, "a/") + target = self.decode_clean(item.target, "b/") + if "dev/null" in source: + target = self.strip_path(target, root, strip) + hunks = [s.decode("utf-8") for s in item.hunks[0].text] + new_file = "".join(hunk[1:] for hunk in hunks) + save(target, new_file) + elif "dev/null" in target: + source = self.strip_path(source, root, strip) + safe_unlink(source) + else: + items.append(item) + self.items = items + + if root: + prevdir = os.getcwd() + os.chdir(root) + + total = len(self.items) + errors = 0 + if strip: + # [ ] test strip level exceeds nesting level + # [ ] test the same only for selected files + # [ ] test if files end up being on the same level + try: + strip = int(strip) + except ValueError: + errors += 1 + warning("error: strip parameter '%s' must be an integer" % strip) + strip = 0 + + #for fileno, filename in enumerate(self.source): + for i,p in enumerate(self.items): + if strip: + debug("stripping %s leading component(s) from:" % strip) + debug(" %s" % p.source) + debug(" %s" % p.target) + old = p.source if p.source == b'/dev/null' else pathstrip(p.source, strip) + new = p.target if p.target == b'/dev/null' else pathstrip(p.target, strip) + else: + old, new = p.source, p.target + + filenameo, filenamen = self.findfiles(old, new) + + if not filenameo or not filenamen: + error("source/target file does not exist:\n --- %s\n +++ %s" % (old, new)) + errors += 1 + continue + if not isfile(filenameo): + error("not a file - %s" % filenameo) + errors += 1 + continue + + # [ ] check absolute paths security here + debug("processing %d/%d:\t %s" % (i+1, total, filenamen)) + + # validate before patching + f2fp = open(filenameo, 'rb') + hunkno = 0 + hunk = p.hunks[hunkno] + hunkfind = [] + hunkreplace = [] + validhunks = 0 + canpatch = False + for lineno, line in enumerate(f2fp): + if lineno+1 < hunk.startsrc: + continue + elif lineno+1 == hunk.startsrc: + hunkfind = [x[1:].rstrip(b"\r\n") for x in hunk.text if x[0] in b" -"] + hunkreplace = [x[1:].rstrip(b"\r\n") for x in hunk.text if x[0] in b" +"] + #pprint(hunkreplace) + hunklineno = 0 + + # todo \ No newline at end of file + + # check hunks in source file + if lineno+1 < hunk.startsrc+len(hunkfind): + if line.rstrip(b"\r\n") == hunkfind[hunklineno]: + hunklineno += 1 + else: + warning("file %d/%d:\t %s" % (i+1, total, filenamen)) + warning(" hunk no.%d doesn't match source file at line %d" % (hunkno+1, lineno+1)) + warning(" expected: %s" % hunkfind[hunklineno]) + warning(" actual : %s" % line.rstrip(b"\r\n")) + if fuzz: + hunklineno += 1 + else: + # not counting this as error, because file may already be patched. + # check if file is already patched is done after the number of + # invalid hunks if found + # TODO: check hunks against source/target file in one pass + # API - check(stream, srchunks, tgthunks) + # return tuple (srcerrs, tgterrs) + + # continue to check other hunks for completeness + hunkno += 1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + continue + else: + break + + # check if processed line is the last line + if len(hunkfind) == 0 or lineno+1 == hunk.startsrc+len(hunkfind)-1: + debug(" hunk no.%d for file %s -- is ready to be patched" % (hunkno+1, filenamen)) + hunkno+=1 + validhunks+=1 + if hunkno < len(p.hunks): + hunk = p.hunks[hunkno] + else: + if validhunks == len(p.hunks): + # patch file + canpatch = True + break + else: + if hunkno < len(p.hunks): + error("premature end of source file %s at hunk %d" % (filenameo, hunkno+1)) + errors += 1 + + f2fp.close() + + if validhunks < len(p.hunks): + if self._match_file_hunks(filenameo, p.hunks): + warning("already patched %s" % filenameo) + else: + if fuzz: + warning("source file is different - %s" % filenameo) + else: + error("source file is different - %s" % filenameo) + errors += 1 + if canpatch: + backupname = filenamen+b".orig" + if exists(backupname): + warning("can't backup original file to %s - aborting" % backupname) + errors += 1 + else: + shutil.move(filenamen, backupname) + if self.write_hunks(backupname if filenameo == filenamen else filenameo, filenamen, p.hunks): + info("successfully patched %d/%d:\t %s" % (i+1, total, filenamen)) + safe_unlink(backupname) + if new == b'/dev/null': + # check that filename is of size 0 and delete it. + if os.path.getsize(filenamen) > 0: + warning("expected patched file to be empty as it's marked as deletion:\t %s" % filenamen) + safe_unlink(filenamen) + else: + errors += 1 + warning("error patching file %s" % filenamen) + shutil.copy(filenamen, filenamen+".invalid") + warning("invalid version is saved to %s" % filenamen+".invalid") + # todo: proper rejects + shutil.move(backupname, filenamen) + + if root: + os.chdir(prevdir) + + # todo: check for premature eof + return (errors == 0) + + + def _reverse(self): + """ reverse patch direction (this doesn't touch filenames) """ + for p in self.items: + for h in p.hunks: + h.startsrc, h.starttgt = h.starttgt, h.startsrc + h.linessrc, h.linestgt = h.linestgt, h.linessrc + for i,line in enumerate(h.text): + # need to use line[0:1] here, because line[0] + # returns int instead of bytes on Python 3 + if line[0:1] == b'+': + h.text[i] = b'-' + line[1:] + elif line[0:1] == b'-': + h.text[i] = b'+' +line[1:] + + def revert(self, strip=0, root=None): + """ apply patch in reverse order """ + reverted = copy.deepcopy(self) + reverted._reverse() + return reverted.apply(strip, root) + + + def can_patch(self, filename): + """ Check if specified filename can be patched. Returns None if file can + not be found among source filenames. False if patch can not be applied + clearly. True otherwise. + + :returns: True, False or None + """ + filename = abspath(filename) + for p in self.items: + if filename == abspath(p.source): + return self._match_file_hunks(filename, p.hunks) + return None + + + def _match_file_hunks(self, filepath, hunks): + matched = True + fp = open(abspath(filepath), 'rb') + + class NoMatch(Exception): + pass + + lineno = 1 + line = fp.readline() + try: + for hno, h in enumerate(hunks): + # skip to first line of the hunk + while lineno < h.starttgt: + if not len(line): # eof + debug("check failed - premature eof before hunk: %d" % (hno+1)) + raise NoMatch + line = fp.readline() + lineno += 1 + for hline in h.text: + if hline.startswith(b"-"): + continue + if not len(line): + debug("check failed - premature eof on hunk: %d" % (hno+1)) + # todo: \ No newline at the end of file + raise NoMatch + if line.rstrip(b"\r\n") != hline[1:].rstrip(b"\r\n"): + debug("file is not patched - failed hunk: %d" % (hno+1)) + raise NoMatch + line = fp.readline() + lineno += 1 + + except NoMatch: + matched = False + # todo: display failed hunk, i.e. expected/found + + fp.close() + return matched + + + def patch_stream(self, instream, hunks): + """ Generator that yields stream patched with hunks iterable + + Converts lineends in hunk lines to the best suitable format + autodetected from input + """ + + # todo: At the moment substituted lineends may not be the same + # at the start and at the end of patching. Also issue a + # warning/throw about mixed lineends (is it really needed?) + + hunks = iter(hunks) + + srclineno = 1 + + lineends = {b'\n':0, b'\r\n':0, b'\r':0} + def get_line(): + """ + local utility function - return line from source stream + collecting line end statistics on the way + """ + line = instream.readline() + # 'U' mode works only with text files + if line.endswith(b"\r\n"): + lineends[b"\r\n"] += 1 + elif line.endswith(b"\n"): + lineends[b"\n"] += 1 + elif line.endswith(b"\r"): + lineends[b"\r"] += 1 + return line + + for hno, h in enumerate(hunks): + debug("hunk %d" % (hno+1)) + # skip to line just before hunk starts + while srclineno < h.startsrc: + yield get_line() + srclineno += 1 + + for hline in h.text: + # todo: check \ No newline at the end of file + if hline.startswith(b"-") or hline.startswith(b"\\"): + get_line() + srclineno += 1 + continue + else: + if not hline.startswith(b"+"): + yield get_line() + srclineno += 1 + continue + line2write = hline[1:] + # detect if line ends are consistent in source file + if sum([bool(lineends[x]) for x in lineends]) == 1: + newline = [x for x in lineends if lineends[x] != 0][0] + yield line2write.rstrip(b"\r\n")+newline + else: # newlines are mixed + yield line2write + + for line in instream: + yield line + + + def write_hunks(self, srcname, tgtname, hunks): + src = open(srcname, "rb") + tgt = open(tgtname, "wb") + + debug("processing target file %s" % tgtname) + + tgt.writelines(self.patch_stream(src, hunks)) + + tgt.close() + src.close() + # [ ] TODO: add test for permission copy + shutil.copymode(srcname, tgtname) + return True + + + def dump(self): + for p in self.items: + for headline in p.header: + print(headline.rstrip('\n')) + print('--- ' + p.source) + print('+++ ' + p.target) + for h in p.hunks: + print('@@ -%s,%s +%s,%s @@' % (h.startsrc, h.linessrc, h.starttgt, h.linestgt)) + for line in h.text: + print(line.rstrip('\n')) + + +def main(): + from optparse import OptionParser + from os.path import exists + import sys + + opt = OptionParser(usage="1. %prog [options] unified.diff\n" + " 2. %prog [options] http://host/patch\n" + " 3. %prog [options] -- < unified.diff", + version="python-patch %s" % __version__) + opt.add_option("-q", "--quiet", action="store_const", dest="verbosity", + const=0, help="print only warnings and errors", default=1) + opt.add_option("-v", "--verbose", action="store_const", dest="verbosity", + const=2, help="be verbose") + opt.add_option("--debug", action="store_true", dest="debugmode", help="debug mode") + opt.add_option("--diffstat", action="store_true", dest="diffstat", + help="print diffstat and exit") + opt.add_option("-d", "--directory", metavar='DIR', + help="specify root directory for applying patch") + opt.add_option("-p", "--strip", type="int", metavar='N', default=0, + help="strip N path components from filenames") + opt.add_option("--revert", action="store_true", + help="apply patch in reverse order (unpatch)") + opt.add_option("-f", "--fuzz", action="store_true", dest="fuzz", help="Accept fuuzzy patches") + (options, args) = opt.parse_args() + + if not args and sys.argv[-1:] != ['--']: + opt.print_version() + opt.print_help() + sys.exit() + readstdin = (sys.argv[-1:] == ['--'] and not args) + + verbosity_levels = {0:logging.WARNING, 1:logging.INFO, 2:logging.DEBUG} + loglevel = verbosity_levels[options.verbosity] + logformat = "%(message)s" + logger.setLevel(loglevel) + streamhandler.setFormatter(logging.Formatter(logformat)) + + if options.debugmode: + setdebug() # this sets global debugmode variable + + if readstdin: + patch = PatchSet(sys.stdin) + else: + patchfile = args[0] + urltest = patchfile.split(':')[0] + if (':' in patchfile and urltest.isalpha() + and len(urltest) > 1): # one char before : is a windows drive letter + patch = fromurl(patchfile) + else: + if not exists(patchfile) or not isfile(patchfile): + sys.exit("patch file does not exist - %s" % patchfile) + patch = fromfile(patchfile) + + if options.diffstat: + print(patch.diffstat()) + sys.exit(0) + + if not patch: + error("Could not parse patch") + sys.exit(-1) + + #pprint(patch) + if options.revert: + patch.revert(options.strip, root=options.directory) or sys.exit(-1) + else: + patch.apply(options.strip, root=options.directory, fuzz=options.fuzz) or sys.exit(-1) + + # todo: document and test line ends handling logic - patch_ng.py detects proper line-endings + # for inserted hunks and issues a warning if patched file has incosistent line ends + + +if __name__ == "__main__": + main() + +# Legend: +# [ ] - some thing to be done +# [w] - official wart, external or internal that is unlikely to be fixed + +# [ ] API break (2.x) wishlist +# PatchSet.items --> PatchSet.patches + +# [ ] run --revert test for all dataset items +# [ ] run .parse() / .dump() test for dataset diff --git a/patches/0001-RadioLib-SPItransfer-virtual.patch b/patches/0001-RadioLib-SPItransfer-virtual.patch new file mode 100644 index 000000000..21aab7e3b --- /dev/null +++ b/patches/0001-RadioLib-SPItransfer-virtual.patch @@ -0,0 +1,12 @@ +index 3a7b098..2492c1a 100644 +--- a/src/Module.h ++++ b/src/Module.h +@@ -190,7 +190,7 @@ class Module { + + \param numBytes Number of bytes to transfer. + */ +- void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t* dataOut, uint8_t* dataIn, uint8_t numBytes); ++ virtual void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t* dataOut, uint8_t* dataIn, uint8_t numBytes); + + // pin number access methods + diff --git a/platformio.ini b/platformio.ini index 5d991fe38..b53bd92ff 100644 --- a/platformio.ini +++ b/platformio.ini @@ -93,6 +93,10 @@ build_src_filter = ${arduino_base.build_src_filter} - upload_speed = 921600 debug_init_break = tbreak setup +extra_scripts = + ${env.extra_scripts} + pre:bin/apply_patches.py + # Remove -DMYNEWT_VAL_BLE_HS_LOG_LVL=LOG_LEVEL_CRITICAL for low level BLE logging. # See library directory for BLE logging possible values: .pio/libdeps/tbeam/NimBLE-Arduino/src/log_common/log_common.h # This overrides the BLE logging default of LOG_LEVEL_INFO (1) from: .pio/libdeps/tbeam/NimBLE-Arduino/src/esp_nimble_cfg.h @@ -144,6 +148,9 @@ build_src_filter = ${arduino_base.build_src_filter} - - - - - - lib_ignore = BluetoothOTA +extra_scripts = + ${env.extra_scripts} + pre:bin/apply_patches.py [nrf52840_base] extends = nrf52_base From 7594140afc96fea28cd9c9d83cf60bf11f2f5554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Sun, 12 Jun 2022 20:31:23 +0200 Subject: [PATCH 02/20] actual change to our interface --- src/mesh/RadioLibInterface.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/RadioLibInterface.h b/src/mesh/RadioLibInterface.h index 35abcc74b..1b07bf37a 100644 --- a/src/mesh/RadioLibInterface.h +++ b/src/mesh/RadioLibInterface.h @@ -54,7 +54,7 @@ class LockingModule : public Module \param numBytes Number of bytes to transfer. */ - virtual void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes); + void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes) override; }; class RadioLibInterface : public RadioInterface, protected concurrency::NotifiedWorkerThread From dc8d1d9a8430179287f11b6f2de449c7d76bd87c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Sun, 12 Jun 2022 21:52:46 +0200 Subject: [PATCH 03/20] implement #1504 --- src/mesh/NodeDB.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index ce3d4d04c..a9ee9df1e 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -215,7 +215,7 @@ void NodeDB::installDefaultDeviceState() // Set default owner name pickNewNodeNum(); // based on macaddr now sprintf(owner.long_name, "Unknown %02x%02x", ourMacAddr[4], ourMacAddr[5]); - sprintf(owner.short_name, "?%02X", (unsigned)(myNodeInfo.my_node_num & 0xff)); + sprintf(owner.short_name, "%02x%02x", ourMacAddr[4], ourMacAddr[5]); sprintf(owner.id, "!%08x", getNodeNum()); // Default node ID now based on nodenum memcpy(owner.macaddr, ourMacAddr, sizeof(owner.macaddr)); From 5c44c4f772545b4a1d9fef96ad4b4f0d45097891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Sun, 12 Jun 2022 23:29:27 +0200 Subject: [PATCH 04/20] Remove Debug Code for Encryption --- src/esp32/ESP32CryptoEngine.cpp | 2 - src/mesh/CryptoEngine.cpp | 78 --------------------------------- src/mesh/CryptoEngine.h | 2 - src/nrf52/NRF52CryptoEngine.cpp | 27 +----------- 4 files changed, 2 insertions(+), 107 deletions(-) diff --git a/src/esp32/ESP32CryptoEngine.cpp b/src/esp32/ESP32CryptoEngine.cpp index e80d59661..2003a235b 100644 --- a/src/esp32/ESP32CryptoEngine.cpp +++ b/src/esp32/ESP32CryptoEngine.cpp @@ -49,7 +49,6 @@ class ESP32CryptoEngine : public CryptoEngine */ virtual void encrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); if (key.length > 0) { uint8_t stream_block[16]; static uint8_t scratch[MAX_BLOCKSIZE]; @@ -65,7 +64,6 @@ class ESP32CryptoEngine : public CryptoEngine auto res = mbedtls_aes_crypt_ctr(&aes, numBytes, &nc_off, nonce, stream_block, scratch, bytes); assert(!res); } - hexDump("after", bytes, numBytes, 16); } virtual void decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override diff --git a/src/mesh/CryptoEngine.cpp b/src/mesh/CryptoEngine.cpp index 8a5ea795d..5e73e3921 100644 --- a/src/mesh/CryptoEngine.cpp +++ b/src/mesh/CryptoEngine.cpp @@ -4,10 +4,6 @@ void CryptoEngine::setKey(const CryptoKey &k) { DEBUG_MSG("Using AES%d key!\n", k.length * 8); - /* for(uint8_t i = 0; i < k.length; i++) - DEBUG_MSG("%02x ", k.bytes[i]); - DEBUG_MSG("\n"); */ - key = k; } @@ -26,78 +22,6 @@ void CryptoEngine::decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes DEBUG_MSG("WARNING: noop decryption!\n"); } -// Usage: -// hexDump(desc, addr, len, perLine); -// desc: if non-NULL, printed as a description before hex dump. -// addr: the address to start dumping from. -// len: the number of bytes to dump. -// perLine: number of bytes on each output line. - -void CryptoEngine::hexDump (const char * desc, const void * addr, const int len, int perLine) -{ - // Silently ignore silly per-line values. - - if (perLine < 4 || perLine > 64) perLine = 16; - - int i; - unsigned char buff[perLine+1]; - const unsigned char * pc = (const unsigned char *)addr; - - // Output description if given. - - if (desc != NULL) DEBUG_MSG ("%s:\n", desc); - - // Length checks. - - if (len == 0) { - DEBUG_MSG(" ZERO LENGTH\n"); - return; - } - if (len < 0) { - DEBUG_MSG(" NEGATIVE LENGTH: %d\n", len); - return; - } - - // Process every byte in the data. - - for (i = 0; i < len; i++) { - // Multiple of perLine means new or first line (with line offset). - - if ((i % perLine) == 0) { - // Only print previous-line ASCII buffer for lines beyond first. - - if (i != 0) DEBUG_MSG (" %s\n", buff); - - // Output the offset of current line. - - DEBUG_MSG (" %04x ", i); - } - - // Now the hex code for the specific character. - - DEBUG_MSG (" %02x", pc[i]); - - // And buffer a printable ASCII character for later. - - if ((pc[i] < 0x20) || (pc[i] > 0x7e)) // isprint() may be better. - buff[i % perLine] = '.'; - else - buff[i % perLine] = pc[i]; - buff[(i % perLine) + 1] = '\0'; - } - - // Pad out last line if not exactly perLine characters. - - while ((i % perLine) != 0) { - DEBUG_MSG (" "); - i++; - } - - // And print the final ASCII buffer. - - DEBUG_MSG (" %s\n", buff); -} - /** * Init our 128 bit nonce for a new packet */ @@ -108,6 +32,4 @@ void CryptoEngine::initNonce(uint32_t fromNode, uint64_t packetId) // use memcpy to avoid breaking strict-aliasing memcpy(nonce, &packetId, sizeof(uint64_t)); memcpy(nonce + sizeof(uint64_t), &fromNode, sizeof(uint32_t)); - //*((uint64_t *)&nonce[0]) = packetId; - //*((uint32_t *)&nonce[8]) = fromNode; } \ No newline at end of file diff --git a/src/mesh/CryptoEngine.h b/src/mesh/CryptoEngine.h index 39b30a727..1dda7ce31 100644 --- a/src/mesh/CryptoEngine.h +++ b/src/mesh/CryptoEngine.h @@ -56,8 +56,6 @@ class CryptoEngine * a 32 bit block counter (starts at zero) */ void initNonce(uint32_t fromNode, uint64_t packetId); - - void hexDump(const char * desc, const void * addr, const int len, int perLine); }; extern CryptoEngine *crypto; diff --git a/src/nrf52/NRF52CryptoEngine.cpp b/src/nrf52/NRF52CryptoEngine.cpp index 287defdda..42eacfc27 100644 --- a/src/nrf52/NRF52CryptoEngine.cpp +++ b/src/nrf52/NRF52CryptoEngine.cpp @@ -16,7 +16,6 @@ class NRF52CryptoEngine : public CryptoEngine */ virtual void encrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); if (key.length > 16) { DEBUG_MSG("Software encrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); AES_ctx ctx; @@ -28,7 +27,6 @@ class NRF52CryptoEngine : public CryptoEngine nRFCrypto.begin(); nRFCrypto_AES ctx; uint8_t myLen = ctx.blockLen(numBytes); - DEBUG_MSG("nRF52 encBuf myLen=%d!\n", myLen); char encBuf[myLen] = {0}; initNonce(fromNode, packetId); ctx.begin(); @@ -37,33 +35,12 @@ class NRF52CryptoEngine : public CryptoEngine nRFCrypto.end(); memcpy(bytes, encBuf, numBytes); } - hexDump("after", bytes, numBytes, 16); } virtual void decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); - if (key.length > 16) { - DEBUG_MSG("Software decrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); - AES_ctx ctx; - initNonce(fromNode, packetId); - AES_init_ctx_iv(&ctx, key.bytes, nonce); - AES_CTR_xcrypt_buffer(&ctx, bytes, numBytes); - } else if (key.length > 0) { - DEBUG_MSG("nRF52 decrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); - nRFCrypto.begin(); - nRFCrypto_AES ctx; - uint8_t myLen = ctx.blockLen(numBytes); - DEBUG_MSG("nRF52 decBuf myLen=%d!\n", myLen); - char decBuf[myLen] = {0}; - initNonce(fromNode, packetId); - ctx.begin(); - ctx.Process((char*)bytes, numBytes, nonce, key.bytes, key.length, decBuf, ctx.decryptFlag, ctx.ctrMode); - ctx.end(); - nRFCrypto.end(); - memcpy(bytes, decBuf, numBytes); - } - hexDump("after", bytes, numBytes, 16); + // For CTR, the implementation is the same + encrypt(fromNode, packetId, numBytes, bytes); } private: From 7bd07db2a8a19308f5d08bb563c7a4e24f52437e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Sun, 12 Jun 2022 23:35:59 +0200 Subject: [PATCH 05/20] Remove nRF Crypt Debug --- src/esp32/ESP32CryptoEngine.cpp | 2 - src/mesh/CryptoEngine.cpp | 78 --------------------------------- src/mesh/CryptoEngine.h | 2 - src/nrf52/NRF52CryptoEngine.cpp | 27 +----------- 4 files changed, 2 insertions(+), 107 deletions(-) diff --git a/src/esp32/ESP32CryptoEngine.cpp b/src/esp32/ESP32CryptoEngine.cpp index e80d59661..2003a235b 100644 --- a/src/esp32/ESP32CryptoEngine.cpp +++ b/src/esp32/ESP32CryptoEngine.cpp @@ -49,7 +49,6 @@ class ESP32CryptoEngine : public CryptoEngine */ virtual void encrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); if (key.length > 0) { uint8_t stream_block[16]; static uint8_t scratch[MAX_BLOCKSIZE]; @@ -65,7 +64,6 @@ class ESP32CryptoEngine : public CryptoEngine auto res = mbedtls_aes_crypt_ctr(&aes, numBytes, &nc_off, nonce, stream_block, scratch, bytes); assert(!res); } - hexDump("after", bytes, numBytes, 16); } virtual void decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override diff --git a/src/mesh/CryptoEngine.cpp b/src/mesh/CryptoEngine.cpp index 8a5ea795d..5e73e3921 100644 --- a/src/mesh/CryptoEngine.cpp +++ b/src/mesh/CryptoEngine.cpp @@ -4,10 +4,6 @@ void CryptoEngine::setKey(const CryptoKey &k) { DEBUG_MSG("Using AES%d key!\n", k.length * 8); - /* for(uint8_t i = 0; i < k.length; i++) - DEBUG_MSG("%02x ", k.bytes[i]); - DEBUG_MSG("\n"); */ - key = k; } @@ -26,78 +22,6 @@ void CryptoEngine::decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes DEBUG_MSG("WARNING: noop decryption!\n"); } -// Usage: -// hexDump(desc, addr, len, perLine); -// desc: if non-NULL, printed as a description before hex dump. -// addr: the address to start dumping from. -// len: the number of bytes to dump. -// perLine: number of bytes on each output line. - -void CryptoEngine::hexDump (const char * desc, const void * addr, const int len, int perLine) -{ - // Silently ignore silly per-line values. - - if (perLine < 4 || perLine > 64) perLine = 16; - - int i; - unsigned char buff[perLine+1]; - const unsigned char * pc = (const unsigned char *)addr; - - // Output description if given. - - if (desc != NULL) DEBUG_MSG ("%s:\n", desc); - - // Length checks. - - if (len == 0) { - DEBUG_MSG(" ZERO LENGTH\n"); - return; - } - if (len < 0) { - DEBUG_MSG(" NEGATIVE LENGTH: %d\n", len); - return; - } - - // Process every byte in the data. - - for (i = 0; i < len; i++) { - // Multiple of perLine means new or first line (with line offset). - - if ((i % perLine) == 0) { - // Only print previous-line ASCII buffer for lines beyond first. - - if (i != 0) DEBUG_MSG (" %s\n", buff); - - // Output the offset of current line. - - DEBUG_MSG (" %04x ", i); - } - - // Now the hex code for the specific character. - - DEBUG_MSG (" %02x", pc[i]); - - // And buffer a printable ASCII character for later. - - if ((pc[i] < 0x20) || (pc[i] > 0x7e)) // isprint() may be better. - buff[i % perLine] = '.'; - else - buff[i % perLine] = pc[i]; - buff[(i % perLine) + 1] = '\0'; - } - - // Pad out last line if not exactly perLine characters. - - while ((i % perLine) != 0) { - DEBUG_MSG (" "); - i++; - } - - // And print the final ASCII buffer. - - DEBUG_MSG (" %s\n", buff); -} - /** * Init our 128 bit nonce for a new packet */ @@ -108,6 +32,4 @@ void CryptoEngine::initNonce(uint32_t fromNode, uint64_t packetId) // use memcpy to avoid breaking strict-aliasing memcpy(nonce, &packetId, sizeof(uint64_t)); memcpy(nonce + sizeof(uint64_t), &fromNode, sizeof(uint32_t)); - //*((uint64_t *)&nonce[0]) = packetId; - //*((uint32_t *)&nonce[8]) = fromNode; } \ No newline at end of file diff --git a/src/mesh/CryptoEngine.h b/src/mesh/CryptoEngine.h index 39b30a727..1dda7ce31 100644 --- a/src/mesh/CryptoEngine.h +++ b/src/mesh/CryptoEngine.h @@ -56,8 +56,6 @@ class CryptoEngine * a 32 bit block counter (starts at zero) */ void initNonce(uint32_t fromNode, uint64_t packetId); - - void hexDump(const char * desc, const void * addr, const int len, int perLine); }; extern CryptoEngine *crypto; diff --git a/src/nrf52/NRF52CryptoEngine.cpp b/src/nrf52/NRF52CryptoEngine.cpp index 287defdda..42eacfc27 100644 --- a/src/nrf52/NRF52CryptoEngine.cpp +++ b/src/nrf52/NRF52CryptoEngine.cpp @@ -16,7 +16,6 @@ class NRF52CryptoEngine : public CryptoEngine */ virtual void encrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); if (key.length > 16) { DEBUG_MSG("Software encrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); AES_ctx ctx; @@ -28,7 +27,6 @@ class NRF52CryptoEngine : public CryptoEngine nRFCrypto.begin(); nRFCrypto_AES ctx; uint8_t myLen = ctx.blockLen(numBytes); - DEBUG_MSG("nRF52 encBuf myLen=%d!\n", myLen); char encBuf[myLen] = {0}; initNonce(fromNode, packetId); ctx.begin(); @@ -37,33 +35,12 @@ class NRF52CryptoEngine : public CryptoEngine nRFCrypto.end(); memcpy(bytes, encBuf, numBytes); } - hexDump("after", bytes, numBytes, 16); } virtual void decrypt(uint32_t fromNode, uint64_t packetId, size_t numBytes, uint8_t *bytes) override { - hexDump("before", bytes, numBytes, 16); - if (key.length > 16) { - DEBUG_MSG("Software decrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); - AES_ctx ctx; - initNonce(fromNode, packetId); - AES_init_ctx_iv(&ctx, key.bytes, nonce); - AES_CTR_xcrypt_buffer(&ctx, bytes, numBytes); - } else if (key.length > 0) { - DEBUG_MSG("nRF52 decrypt fr=%x, num=%x, numBytes=%d!\n", fromNode, (uint32_t) packetId, numBytes); - nRFCrypto.begin(); - nRFCrypto_AES ctx; - uint8_t myLen = ctx.blockLen(numBytes); - DEBUG_MSG("nRF52 decBuf myLen=%d!\n", myLen); - char decBuf[myLen] = {0}; - initNonce(fromNode, packetId); - ctx.begin(); - ctx.Process((char*)bytes, numBytes, nonce, key.bytes, key.length, decBuf, ctx.decryptFlag, ctx.ctrMode); - ctx.end(); - nRFCrypto.end(); - memcpy(bytes, decBuf, numBytes); - } - hexDump("after", bytes, numBytes, 16); + // For CTR, the implementation is the same + encrypt(fromNode, packetId, numBytes, bytes); } private: From 553b35d0ad5d74beccc4796aac9112ac85b82061 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Sun, 12 Jun 2022 19:56:32 -0500 Subject: [PATCH 06/20] Consolidate power saving prefs (#1507) * Fixed conversion linter warning * Power saving consolidation * Whoops --- protobufs | 2 +- src/PowerFSM.cpp | 4 ++-- src/mesh/NodeDB.h | 4 ++-- src/mesh/generated/config.pb.h | 15 ++++++--------- src/mesh/generated/localonly.pb.h | 2 +- 5 files changed, 12 insertions(+), 15 deletions(-) diff --git a/protobufs b/protobufs index e5b5adc19..8f616d3ee 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit e5b5adc196d3593ab15c04101093443ab8b36c9c +Subproject commit 8f616d3eefd4edc86b915f3d7fae827847bdcd95 diff --git a/src/PowerFSM.cpp b/src/PowerFSM.cpp index 82f73147c..bdfbda87c 100644 --- a/src/PowerFSM.cpp +++ b/src/PowerFSM.cpp @@ -20,14 +20,14 @@ static bool isPowered() // If we are not a router and we already have AC power go to POWER state after init, otherwise go to ON // We assume routers might be powered all the time, but from a low current (solar) source - bool isLowPower = config.power.is_low_power || isRouter; + bool isPowerSavingMode = config.power.is_power_saving || isRouter; /* To determine if we're externally powered, assumptions 1) If we're powered up and there's no battery, we must be getting power externally. (because we'd be dead otherwise) 2) If we detect USB power from the power management chip, we must be getting power externally. */ - return !isLowPower && powerStatus && (!powerStatus->getHasBattery() || powerStatus->getHasUSB()); + return !isPowerSavingMode && powerStatus && (!powerStatus->getHasBattery() || powerStatus->getHasUSB()); } static void sdsEnter() diff --git a/src/mesh/NodeDB.h b/src/mesh/NodeDB.h index 0d6d50ee1..2cd3a8331 100644 --- a/src/mesh/NodeDB.h +++ b/src/mesh/NodeDB.h @@ -138,7 +138,7 @@ extern NodeDB nodeDB; /* If is_router is set, we use a number of different default values - # FIXME - after tuning, move these params into the on-device defaults based on is_router and is_low_power + # FIXME - after tuning, move these params into the on-device defaults based on is_router and is_power_saving # prefs.position_broadcast_secs = FIXME possibly broadcast only once an hr prefs.wait_bluetooth_secs = 1 # Don't stay in bluetooth mode @@ -152,7 +152,7 @@ extern NodeDB nodeDB; # get a new GPS position once per day prefs.gps_update_interval = oneday - prefs.is_low_power = True + prefs.is_power_saving = True # allow up to five minutes for each new GPS lock attempt prefs.gps_attempt_time = 300 diff --git a/src/mesh/generated/config.pb.h b/src/mesh/generated/config.pb.h index e59fda04b..45724e61b 100644 --- a/src/mesh/generated/config.pb.h +++ b/src/mesh/generated/config.pb.h @@ -125,10 +125,9 @@ typedef struct _Config_PositionConfig { typedef struct _Config_PowerConfig { Config_PowerConfig_ChargeCurrent charge_current; - bool is_low_power; + bool is_power_saving; bool is_always_powered; uint32_t on_battery_shutdown_after_secs; - bool is_power_saving; float adc_multiplier_override; uint32_t wait_bluetooth_secs; uint32_t mesh_sds_timeout_secs; @@ -192,14 +191,14 @@ extern "C" { #define Config_init_default {0, {Config_DeviceConfig_init_default}} #define Config_DeviceConfig_init_default {_Config_DeviceConfig_Role_MIN, 0, 0, 0, ""} #define Config_PositionConfig_init_default {0, 0, 0, 0, 0, 0, 0} -#define Config_PowerConfig_init_default {_Config_PowerConfig_ChargeCurrent_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +#define Config_PowerConfig_init_default {_Config_PowerConfig_ChargeCurrent_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0} #define Config_WiFiConfig_init_default {"", "", 0, 0} #define Config_DisplayConfig_init_default {0, _Config_DisplayConfig_GpsCoordinateFormat_MIN, 0} #define Config_LoRaConfig_init_default {0, _Config_LoRaConfig_ModemPreset_MIN, 0, 0, 0, 0, _Config_LoRaConfig_RegionCode_MIN, 0, 0, 0, {0, 0, 0}} #define Config_init_zero {0, {Config_DeviceConfig_init_zero}} #define Config_DeviceConfig_init_zero {_Config_DeviceConfig_Role_MIN, 0, 0, 0, ""} #define Config_PositionConfig_init_zero {0, 0, 0, 0, 0, 0, 0} -#define Config_PowerConfig_init_zero {_Config_PowerConfig_ChargeCurrent_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} +#define Config_PowerConfig_init_zero {_Config_PowerConfig_ChargeCurrent_MIN, 0, 0, 0, 0, 0, 0, 0, 0, 0} #define Config_WiFiConfig_init_zero {"", "", 0, 0} #define Config_DisplayConfig_init_zero {0, _Config_DisplayConfig_GpsCoordinateFormat_MIN, 0} #define Config_LoRaConfig_init_zero {0, _Config_LoRaConfig_ModemPreset_MIN, 0, 0, 0, 0, _Config_LoRaConfig_RegionCode_MIN, 0, 0, 0, {0, 0, 0}} @@ -231,10 +230,9 @@ extern "C" { #define Config_PositionConfig_gps_attempt_time_tag 7 #define Config_PositionConfig_position_flags_tag 10 #define Config_PowerConfig_charge_current_tag 1 -#define Config_PowerConfig_is_low_power_tag 2 +#define Config_PowerConfig_is_power_saving_tag 2 #define Config_PowerConfig_is_always_powered_tag 3 #define Config_PowerConfig_on_battery_shutdown_after_secs_tag 4 -#define Config_PowerConfig_is_power_saving_tag 5 #define Config_PowerConfig_adc_multiplier_override_tag 6 #define Config_PowerConfig_wait_bluetooth_secs_tag 7 #define Config_PowerConfig_mesh_sds_timeout_secs_tag 9 @@ -291,10 +289,9 @@ X(a, STATIC, SINGULAR, UINT32, position_flags, 10) #define Config_PowerConfig_FIELDLIST(X, a) \ X(a, STATIC, SINGULAR, UENUM, charge_current, 1) \ -X(a, STATIC, SINGULAR, BOOL, is_low_power, 2) \ +X(a, STATIC, SINGULAR, BOOL, is_power_saving, 2) \ X(a, STATIC, SINGULAR, BOOL, is_always_powered, 3) \ X(a, STATIC, SINGULAR, UINT32, on_battery_shutdown_after_secs, 4) \ -X(a, STATIC, SINGULAR, BOOL, is_power_saving, 5) \ X(a, STATIC, SINGULAR, FLOAT, adc_multiplier_override, 6) \ X(a, STATIC, SINGULAR, UINT32, wait_bluetooth_secs, 7) \ X(a, STATIC, SINGULAR, UINT32, mesh_sds_timeout_secs, 9) \ @@ -355,7 +352,7 @@ extern const pb_msgdesc_t Config_LoRaConfig_msg; #define Config_DisplayConfig_size 14 #define Config_LoRaConfig_size 67 #define Config_PositionConfig_size 30 -#define Config_PowerConfig_size 49 +#define Config_PowerConfig_size 47 #define Config_WiFiConfig_size 103 #define Config_size 105 diff --git a/src/mesh/generated/localonly.pb.h b/src/mesh/generated/localonly.pb.h index df0966414..06922a88d 100644 --- a/src/mesh/generated/localonly.pb.h +++ b/src/mesh/generated/localonly.pb.h @@ -126,7 +126,7 @@ extern const pb_msgdesc_t LocalModuleConfig_msg; #define LocalModuleConfig_fields &LocalModuleConfig_msg /* Maximum encoded size of messages (where known) */ -#define LocalConfig_size 317 +#define LocalConfig_size 315 #define LocalModuleConfig_size 282 #ifdef __cplusplus From a1b07ed6aa6923b04b3f3e16b385b5ce8131de17 Mon Sep 17 00:00:00 2001 From: GUVWAF <78759985+GUVWAF@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:21:18 +0200 Subject: [PATCH 07/20] Introduce contention window (CW) (#1500) Co-authored-by: Ben Meadors --- src/mesh/RadioInterface.cpp | 58 ++++++++++++++-------------------- src/mesh/RadioInterface.h | 16 ++++++---- src/mesh/RadioLibInterface.cpp | 16 +++------- 3 files changed, 38 insertions(+), 52 deletions(-) diff --git a/src/mesh/RadioInterface.cpp b/src/mesh/RadioInterface.cpp index d949c4626..635862956 100644 --- a/src/mesh/RadioInterface.cpp +++ b/src/mesh/RadioInterface.cpp @@ -159,61 +159,50 @@ uint32_t RadioInterface::getPacketTime(MeshPacket *p) /** The delay to use for retransmitting dropped packets */ uint32_t RadioInterface::getRetransmissionMsec(const MeshPacket *p) { - assert(shortPacketMsec); // Better be non zero + assert(slotTimeMsec); // Better be non zero static uint8_t bytes[MAX_RHPACKETLEN]; size_t numbytes = pb_encode_to_bytes(bytes, sizeof(bytes), Data_fields, &p->decoded); uint32_t packetAirtime = getPacketTime(numbytes + sizeof(PacketHeader)); - uint32_t tCADmsec = 2 * (1 << sf) / bw; // duration of CAD is roughly 2 symbols according to SX127x datasheet - /* Make sure enough time has elapsed for this packet to be sent and an ACK is received. - * Right now we have to wait until another node floods the same packet, as that is our implicit ACK. - * TODO: Revise when want_ack will be used (right now it is always set to 0 afterwards). - */ - return 2*packetAirtime + 2*MIN_TX_WAIT_MSEC + shortPacketMsec + shortPacketMsec*2 + PROCESSING_TIME_MSEC + 2*tCADmsec; + // Make sure enough time has elapsed for this packet to be sent and an ACK is received. + // DEBUG_MSG("Waiting for flooding message with airtime %d and slotTime is %d\n", packetAirtime, slotTimeMsec); + float channelUtil = airTime->channelUtilizationPercent(); + uint8_t CWsize = map(channelUtil, 0, 100, CWmin, CWmax); + // Assuming we pick max. of CWsize and there will be a receiver with SNR at half the range + return 2*packetAirtime + (pow(2, CWsize) + pow(2, int((CWmax+CWmin)/2))) * slotTimeMsec + PROCESSING_TIME_MSEC; } -/** The delay to use when we want to send something but the ether is busy */ +/** The delay to use when we want to send something */ uint32_t RadioInterface::getTxDelayMsec() { - /** At the low end we want to pick a delay large enough that anyone who just completed sending (some other node) - * has had enough time to switch their radio back into receive mode. - */ - const uint32_t MIN_TX_WAIT_MSEC = 100; - - /** - * At the high end, this value is used to spread node attempts across time so when they are replying to a packet - * they don't both check that the airwaves are clear at the same moment. As long as they are off by some amount - * one of the two will be first to start transmitting and the other will see that. I bet 500ms is more than enough - * to guarantee this. - */ - // const uint32_t MAX_TX_WAIT_MSEC = 2000; // stress test would still fail occasionally with 1000 - - return random((MIN_TX_WAIT_MSEC), (MIN_TX_WAIT_MSEC + shortPacketMsec)); + /** We wait a random multiple of 'slotTimes' (see definition in header file) in order to avoid collisions. + The pool to take a random multiple from is the contention window (CW), which size depends on the + current channel utilization. */ + float channelUtil = airTime->channelUtilizationPercent(); + uint8_t CWsize = map(channelUtil, 0, 100, CWmin, CWmax); + // DEBUG_MSG("Current channel utilization is %f so setting CWsize to %d\n", channelUtil, CWsize); + return random(0, pow(2, CWsize)) * slotTimeMsec; } -/** The delay to use when we want to send something but the ether is busy */ +/** The delay to use when we want to flood a message */ uint32_t RadioInterface::getTxDelayMsecWeighted(float snr) { - /** At the low end we want to pick a delay large enough that anyone who just completed sending (some other node) - * has had enough time to switch their radio back into receive mode. - */ - const uint32_t MIN_TX_WAIT_MSEC = 100; - // The minimum value for a LoRa SNR const uint32_t SNR_MIN = -20; // The maximum value for a LoRa SNR const uint32_t SNR_MAX = 15; - // high SNR = Long Delay - // low SNR = Short Delay + // high SNR = large CW size (Long Delay) + // low SNR = small CW size (Short Delay) uint32_t delay = 0; - + uint8_t CWsize = map(snr, SNR_MIN, SNR_MAX, CWmin, CWmax); + // DEBUG_MSG("rx_snr of %f so setting CWsize to:%d\n", snr, CWsize); if (config.device.role == Config_DeviceConfig_Role_Router || config.device.role == Config_DeviceConfig_Role_RouterClient) { - delay = map(snr, SNR_MIN, SNR_MAX, MIN_TX_WAIT_MSEC, (MIN_TX_WAIT_MSEC + (shortPacketMsec / 2))); + delay = random(0, 2*CWsize) * slotTimeMsec; DEBUG_MSG("rx_snr found in packet. As a router, setting tx delay:%d\n", delay); } else { - delay = map(snr, SNR_MIN, SNR_MAX, MIN_TX_WAIT_MSEC + (shortPacketMsec / 2), (MIN_TX_WAIT_MSEC + shortPacketMsec * 2)); + delay = random(0, pow(2, CWsize)) * slotTimeMsec; DEBUG_MSG("rx_snr found in packet. Setting tx delay:%d\n", delay); } @@ -411,7 +400,6 @@ void RadioInterface::applyModemConfig() } power = loraConfig.tx_power; - shortPacketMsec = getPacketTime(sizeof(PacketHeader)); assert(myRegion); // Should have been found in init // Calculate the number of channels @@ -431,7 +419,7 @@ void RadioInterface::applyModemConfig() DEBUG_MSG("Radio myRegion->numChannels: %d\n", numChannels); DEBUG_MSG("Radio channel_num: %d\n", channel_num); DEBUG_MSG("Radio frequency: %f\n", getFreq()); - DEBUG_MSG("Short packet time: %u msec\n", shortPacketMsec); + DEBUG_MSG("Slot time: %u msec\n", slotTimeMsec); } /** diff --git a/src/mesh/RadioInterface.h b/src/mesh/RadioInterface.h index c5ff642c9..898ef20e4 100644 --- a/src/mesh/RadioInterface.h +++ b/src/mesh/RadioInterface.h @@ -52,8 +52,6 @@ class RadioInterface CallbackObserver notifyDeepSleepObserver = CallbackObserver(this, &RadioInterface::notifyDeepSleepCb); - /// Number of msecs we expect our shortest actual packet to be over the wire (used in retry timeout calcs) - uint32_t shortPacketMsec; protected: bool disabled = false; @@ -61,10 +59,16 @@ class RadioInterface float bw = 125; uint8_t sf = 9; uint8_t cr = 7; - + /** Slottime is the minimum time to wait, consisting of: + - CAD duration (maximum of SX126x and SX127x); + - roundtrip air propagation time (assuming max. 30km between nodes); + - Tx/Rx turnaround time (maximum of SX126x and SX127x); + - MAC processing time (measured on T-beam) */ + uint32_t slotTimeMsec = 8.5 * pow(2, sf)/bw + 0.2 + 0.4 + 7; uint16_t preambleLength = 32; // 8 is default, but we use longer to increase the amount of sleep time when receiving - const uint32_t MIN_TX_WAIT_MSEC = 100; // minimum time to wait before transmitting after sensing the channel in ms const uint32_t PROCESSING_TIME_MSEC = 4500; // time to construct, process and construct a packet again (empirically determined) + const uint8_t CWmin = 2; // minimum CWsize + const uint8_t CWmax = 8; // maximum CWsize MeshPacket *sendingPacket = NULL; // The packet we are currently sending uint32_t lastTxStart = 0L; @@ -128,10 +132,10 @@ class RadioInterface /** The delay to use for retransmitting dropped packets */ uint32_t getRetransmissionMsec(const MeshPacket *p); - /** The delay to use when we want to send something but the ether is busy */ + /** The delay to use when we want to send something */ uint32_t getTxDelayMsec(); - /** The delay to use when we want to send something but the ether is busy. Use a weighted scale based on SNR */ + /** The delay to use when we want to flood a message. Use a weighted scale based on SNR */ uint32_t getTxDelayMsecWeighted(float snr); diff --git a/src/mesh/RadioLibInterface.cpp b/src/mesh/RadioLibInterface.cpp index 5d049a291..afe176656 100644 --- a/src/mesh/RadioLibInterface.cpp +++ b/src/mesh/RadioLibInterface.cpp @@ -170,17 +170,11 @@ ErrorCode RadioLibInterface::send(MeshPacket *p) } /** radio helper thread callback. - - We never immediately transmit after any operation (either rx or tx). Instead we should start receiving and - wait a random delay of 100ms to 100ms+shortPacketMsec to make sure we are not stomping on someone else. The 100ms delay - at the beginning ensures all possible listeners have had time to finish processing the previous packet and now have their - radio in RX state. The up to 100ms+shortPacketMsec random delay gives a chance for all possible senders to have high odds - of detecting that someone else started transmitting first and then they will wait until that packet finishes. - - NOTE: the large flood rebroadcast delay might still be needed even with this approach. Because we might not be able to - hear other transmitters that we are potentially stomping on. Requires further thought. - - FIXME, the MIN_TX_WAIT_MSEC and MAX_TX_WAIT_MSEC values should be tuned via logic analyzer later. + We never immediately transmit after any operation (either Rx or Tx). Instead we should wait a random multiple of + 'slotTimes' (see definition in RadioInterface.h) taken from a contention window (CW) to lower the chance of collision. + The CW size is determined by setTransmitDelay() and depends either on the current channel utilization or SNR in case + of a flooding message. After this, we perform channel activity detection (CAD) and reset the transmit delay if it is + currently active. */ void RadioLibInterface::onNotify(uint32_t notification) { From 6b8afdadc22502ca51abe02784dfb69a5f71c990 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 13 Jun 2022 16:10:16 +0200 Subject: [PATCH 08/20] New variant of Radiolib patch --- .../0001-RadioLib-SPItransfer-virtual.patch | 21 +++++++++++-------- src/mesh/RadioLibInterface.cpp | 13 +++++++++--- src/mesh/RadioLibInterface.h | 16 ++------------ 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/patches/0001-RadioLib-SPItransfer-virtual.patch b/patches/0001-RadioLib-SPItransfer-virtual.patch index 21aab7e3b..07eb31f77 100644 --- a/patches/0001-RadioLib-SPItransfer-virtual.patch +++ b/patches/0001-RadioLib-SPItransfer-virtual.patch @@ -1,12 +1,15 @@ -index 3a7b098..2492c1a 100644 +index 3a7b098..aa38f6d 100644 --- a/src/Module.h +++ b/src/Module.h -@@ -190,7 +190,7 @@ class Module { - - \param numBytes Number of bytes to transfer. - */ -- void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t* dataOut, uint8_t* dataIn, uint8_t numBytes); -+ virtual void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t* dataOut, uint8_t* dataIn, uint8_t numBytes); - - // pin number access methods +@@ -361,9 +361,9 @@ class Module { + // helper functions to set up SPI overrides on Arduino + #if defined(RADIOLIB_BUILD_ARDUINO) + void SPIbegin(); +- void SPIbeginTransaction(); ++ virtual void SPIbeginTransaction(); + uint8_t SPItransfer(uint8_t b); +- void SPIendTransaction(); ++ virtual void SPIendTransaction(); + void SPIend(); + #endif diff --git a/src/mesh/RadioLibInterface.cpp b/src/mesh/RadioLibInterface.cpp index afe176656..afec06ba0 100644 --- a/src/mesh/RadioLibInterface.cpp +++ b/src/mesh/RadioLibInterface.cpp @@ -11,11 +11,18 @@ // FIXME, we default to 4MHz SPI, SPI mode 0, check if the datasheet says it can really do that static SPISettings spiSettings(4000000, MSBFIRST, SPI_MODE0); -void LockingModule::SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes) +void LockingModule::SPIbeginTransaction() { - concurrency::LockGuard g(spiLock); + spiLock->lock(); - Module::SPItransfer(cmd, reg, dataOut, dataIn, numBytes); + Module::SPIbeginTransaction(); +} + +void LockingModule::SPIendTransaction() +{ + spiLock->unlock(); + + Module::SPIendTransaction(); } RadioLibInterface::RadioLibInterface(RADIOLIB_PIN_TYPE cs, RADIOLIB_PIN_TYPE irq, RADIOLIB_PIN_TYPE rst, RADIOLIB_PIN_TYPE busy, diff --git a/src/mesh/RadioLibInterface.h b/src/mesh/RadioLibInterface.h index 35abcc74b..f940f8908 100644 --- a/src/mesh/RadioLibInterface.h +++ b/src/mesh/RadioLibInterface.h @@ -41,20 +41,8 @@ class LockingModule : public Module { } - /*! - \brief SPI single transfer method. - - \param cmd SPI access command (read/write/burst/...). - - \param reg Address of SPI register to transfer to/from. - - \param dataOut Data that will be transfered from master to slave. - - \param dataIn Data that was transfered from slave to master. - - \param numBytes Number of bytes to transfer. - */ - virtual void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes); + void SPIbeginTransaction() override; + void SPIendTransaction() override; }; class RadioLibInterface : public RadioInterface, protected concurrency::NotifiedWorkerThread From c9822dee9315c70502717b84f952e9c1ebe3f18f Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Mon, 13 Jun 2022 09:38:33 -0700 Subject: [PATCH 09/20] Update to 1.13.17 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index fd7f6e0d4..dd4911456 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 16 +build = 17 From 6e671d808a2b6b7aaf9e460278ca808e3bb83764 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 13 Jun 2022 21:25:27 +0200 Subject: [PATCH 10/20] Our mod was accepted by RadioLib --- bin/apply_patches.py | 27 - bin/patch_ng.py | 1381 ----------------- .../0001-RadioLib-SPItransfer-virtual.patch | 15 - platformio.ini | 9 +- 4 files changed, 2 insertions(+), 1430 deletions(-) delete mode 100644 bin/apply_patches.py delete mode 100644 bin/patch_ng.py delete mode 100644 patches/0001-RadioLib-SPItransfer-virtual.patch diff --git a/bin/apply_patches.py b/bin/apply_patches.py deleted file mode 100644 index d8c183b05..000000000 --- a/bin/apply_patches.py +++ /dev/null @@ -1,27 +0,0 @@ -from os.path import join, isfile - -Import("env") - -LIBRARY_DIR = join (env["PROJECT_LIBDEPS_DIR"], env["PIOENV"], "RadioLib") -patchflag_path = join(LIBRARY_DIR, ".patching-done") -patch = join(env["PROJECT_DIR"], "bin", "patch_ng.py") - -# patch file only if we didn't do it before -if not isfile(join(LIBRARY_DIR, ".patching-done")): - original_path = join(LIBRARY_DIR) - patch_file = join(env["PROJECT_DIR"], "patches", "0001-RadioLib-SPItransfer-virtual.patch") - - assert isfile(patch_file) - - env.Execute( - env.VerboseAction( - "$PYTHONEXE %s -p 1 --directory=%s %s" % (patch, original_path, patch_file) - , "Applying patch to RadioLib" - ) - ) - - def _touch(path): - with open(path, "w") as fp: - fp.write("") - - env.Execute(lambda *args, **kwargs: _touch(patchflag_path)) \ No newline at end of file diff --git a/bin/patch_ng.py b/bin/patch_ng.py deleted file mode 100644 index 088fbbab3..000000000 --- a/bin/patch_ng.py +++ /dev/null @@ -1,1381 +0,0 @@ -#!/usr/bin/env python -""" - Patch utility to apply unified diffs - - Brute-force line-by-line non-recursive parsing - - Copyright (c) 2008-2016 anatoly techtonik - Available under the terms of MIT license - ---- - The MIT License (MIT) - - Copyright (c) 2019 JFrog LTD - - Permission is hereby granted, free of charge, to any person obtaining a copy of this software - and associated documentation files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all copies or - substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, - INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR - ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, - ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. -""" -from __future__ import print_function - -__author__ = "Conan.io " -__version__ = "1.17.4" -__license__ = "MIT" -__url__ = "https://github.com/conan-io/python-patch" - -import copy -import logging -import re -import tempfile -import codecs - -# cStringIO doesn't support unicode in 2.5 -try: - from StringIO import StringIO -except ImportError: - from io import BytesIO as StringIO # python 3 -try: - import urllib2 as urllib_request -except ImportError: - import urllib.request as urllib_request - -from os.path import exists, isfile, abspath -import os -import posixpath -import shutil -import sys -import stat - - -PY3K = sys.version_info >= (3, 0) - -# PEP 3114 -if not PY3K: - compat_next = lambda gen: gen.next() -else: - compat_next = lambda gen: gen.__next__() - -def tostr(b): - """ Python 3 bytes encoder. Used to print filename in - diffstat output. Assumes that filenames are in utf-8. - """ - if not PY3K: - return b - - # [ ] figure out how to print non-utf-8 filenames without - # information loss - return b.decode('utf-8') - - -#------------------------------------------------ -# Logging is controlled by logger named after the -# module name (e.g. 'patch' for patch_ng.py module) - -logger = logging.getLogger("patch_ng") - -debug = logger.debug -info = logger.info -warning = logger.warning -error = logger.error - -class NullHandler(logging.Handler): - """ Copied from Python 2.7 to avoid getting - `No handlers could be found for logger "patch"` - http://bugs.python.org/issue16539 - """ - def handle(self, record): - pass - def emit(self, record): - pass - def createLock(self): - self.lock = None - -streamhandler = logging.StreamHandler() - -# initialize logger itself -logger.addHandler(NullHandler()) - -debugmode = False - -def setdebug(): - global debugmode, streamhandler - - debugmode = True - loglevel = logging.DEBUG - logformat = "%(levelname)8s %(message)s" - logger.setLevel(loglevel) - - if streamhandler not in logger.handlers: - # when used as a library, streamhandler is not added - # by default - logger.addHandler(streamhandler) - - streamhandler.setFormatter(logging.Formatter(logformat)) - - -#------------------------------------------------ -# Constants for Patch/PatchSet types - -DIFF = PLAIN = "plain" -GIT = "git" -HG = MERCURIAL = "mercurial" -SVN = SUBVERSION = "svn" -# mixed type is only actual when PatchSet contains -# Patches of different type -MIXED = MIXED = "mixed" - - -#------------------------------------------------ -# Helpers (these could come with Python stdlib) - -# x...() function are used to work with paths in -# cross-platform manner - all paths use forward -# slashes even on Windows. - -def xisabs(filename): - """ Cross-platform version of `os.path.isabs()` - Returns True if `filename` is absolute on - Linux, OS X or Windows. - """ - if filename.startswith(b'/'): # Linux/Unix - return True - elif filename.startswith(b'\\'): # Windows - return True - elif re.match(b'\\w:[\\\\/]', filename): # Windows - return True - return False - -def xnormpath(path): - """ Cross-platform version of os.path.normpath """ - # replace escapes and Windows slashes - normalized = posixpath.normpath(path).replace(b'\\', b'/') - # fold the result - return posixpath.normpath(normalized) - -def xstrip(filename): - """ Make relative path out of absolute by stripping - prefixes used on Linux, OS X and Windows. - - This function is critical for security. - """ - while xisabs(filename): - # strip windows drive with all slashes - if re.match(b'\\w:[\\\\/]', filename): - filename = re.sub(b'^\\w+:[\\\\/]+', b'', filename) - # strip all slashes - elif re.match(b'[\\\\/]', filename): - filename = re.sub(b'^[\\\\/]+', b'', filename) - return filename - - -def safe_unlink(filepath): - os.chmod(filepath, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) - os.unlink(filepath) - - -#----------------------------------------------- -# Main API functions - -def fromfile(filename): - """ Parse patch file. If successful, returns - PatchSet() object. Otherwise returns False. - """ - patchset = PatchSet() - debug("reading %s" % filename) - fp = open(filename, "rb") - res = patchset.parse(fp) - fp.close() - if res == True: - return patchset - return False - - -def fromstring(s): - """ Parse text string and return PatchSet() - object (or False if parsing fails) - """ - ps = PatchSet( StringIO(s) ) - if ps.errors == 0: - return ps - return False - - -def fromurl(url): - """ Parse patch from an URL, return False - if an error occured. Note that this also - can throw urlopen() exceptions. - """ - ps = PatchSet( urllib_request.urlopen(url) ) - if ps.errors == 0: - return ps - return False - - -# --- Utility functions --- -# [ ] reuse more universal pathsplit() -def pathstrip(path, n): - """ Strip n leading components from the given path """ - pathlist = [path] - while os.path.dirname(pathlist[0]) != b'': - pathlist[0:1] = os.path.split(pathlist[0]) - return b'/'.join(pathlist[n:]) -# --- /Utility function --- - - -def decode_text(text): - encodings = {codecs.BOM_UTF8: "utf_8_sig", - codecs.BOM_UTF16_BE: "utf_16_be", - codecs.BOM_UTF16_LE: "utf_16_le", - codecs.BOM_UTF32_BE: "utf_32_be", - codecs.BOM_UTF32_LE: "utf_32_le", - b'\x2b\x2f\x76\x38': "utf_7", - b'\x2b\x2f\x76\x39': "utf_7", - b'\x2b\x2f\x76\x2b': "utf_7", - b'\x2b\x2f\x76\x2f': "utf_7", - b'\x2b\x2f\x76\x38\x2d': "utf_7"} - for bom in sorted(encodings, key=len, reverse=True): - if text.startswith(bom): - try: - return text[len(bom):].decode(encodings[bom]) - except UnicodeDecodeError: - continue - decoders = ["utf-8", "Windows-1252"] - for decoder in decoders: - try: - return text.decode(decoder) - except UnicodeDecodeError: - continue - logger.warning("can't decode %s" % str(text)) - return text.decode("utf-8", "ignore") # Ignore not compatible characters - - -def to_file_bytes(content): - if PY3K: - if not isinstance(content, bytes): - content = bytes(content, "utf-8") - elif isinstance(content, unicode): - content = content.encode("utf-8") - return content - - -def load(path, binary=False): - """ Loads a file content """ - with open(path, 'rb') as handle: - tmp = handle.read() - return tmp if binary else decode_text(tmp) - - -def save(path, content, only_if_modified=False): - """ - Saves a file with given content - Params: - path: path to write file to - content: contents to save in the file - only_if_modified: file won't be modified if the content hasn't changed - """ - try: - os.makedirs(os.path.dirname(path)) - except Exception: - pass - - new_content = to_file_bytes(content) - - if only_if_modified and os.path.exists(path): - old_content = load(path, binary=True) - if old_content == new_content: - return - - with open(path, "wb") as handle: - handle.write(new_content) - - -class Hunk(object): - """ Parsed hunk data container (hunk starts with @@ -R +R @@) """ - - def __init__(self): - self.startsrc=None #: line count starts with 1 - self.linessrc=None - self.starttgt=None - self.linestgt=None - self.invalid=False - self.desc='' - self.text=[] - - -class Patch(object): - """ Patch for a single file. - If used as an iterable, returns hunks. - """ - def __init__(self): - self.source = None - self.target = None - self.hunks = [] - self.hunkends = [] - self.header = [] - - self.type = None - - def __iter__(self): - for h in self.hunks: - yield h - - -class PatchSet(object): - """ PatchSet is a patch parser and container. - When used as an iterable, returns patches. - """ - - def __init__(self, stream=None): - # --- API accessible fields --- - - # name of the PatchSet (filename or ...) - self.name = None - # patch set type - one of constants - self.type = None - - # list of Patch objects - self.items = [] - - self.errors = 0 # fatal parsing errors - self.warnings = 0 # non-critical warnings - # --- /API --- - - if stream: - self.parse(stream) - - def __len__(self): - return len(self.items) - - def __iter__(self): - for i in self.items: - yield i - - def parse(self, stream): - """ parse unified diff - return True on success - """ - lineends = dict(lf=0, crlf=0, cr=0) - nexthunkno = 0 #: even if index starts with 0 user messages number hunks from 1 - - p = None - hunk = None - # hunkactual variable is used to calculate hunk lines for comparison - hunkactual = dict(linessrc=None, linestgt=None) - - - class wrapumerate(enumerate): - """Enumerate wrapper that uses boolean end of stream status instead of - StopIteration exception, and properties to access line information. - """ - - def __init__(self, *args, **kwargs): - # we don't call parent, it is magically created by __new__ method - - self._exhausted = False - self._lineno = False # after end of stream equal to the num of lines - self._line = False # will be reset to False after end of stream - - def next(self): - """Try to read the next line and return True if it is available, - False if end of stream is reached.""" - if self._exhausted: - return False - - try: - self._lineno, self._line = compat_next(super(wrapumerate, self)) - except StopIteration: - self._exhausted = True - self._line = False - return False - return True - - @property - def is_empty(self): - return self._exhausted - - @property - def line(self): - return self._line - - @property - def lineno(self): - return self._lineno - - # define states (possible file regions) that direct parse flow - headscan = True # start with scanning header - filenames = False # lines starting with --- and +++ - - hunkhead = False # @@ -R +R @@ sequence - hunkbody = False # - hunkskip = False # skipping invalid hunk mode - - hunkparsed = False # state after successfully parsed hunk - - # regexp to match start of hunk, used groups - 1,3,4,6 - re_hunk_start = re.compile(b"^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))? @@") - - self.errors = 0 - # temp buffers for header and filenames info - header = [] - srcname = None - tgtname = None - - # start of main cycle - # each parsing block already has line available in fe.line - fe = wrapumerate(stream) - while fe.next(): - - # -- deciders: these only switch state to decide who should process - # -- line fetched at the start of this cycle - if hunkparsed: - hunkparsed = False - if re_hunk_start.match(fe.line): - hunkhead = True - elif fe.line.startswith(b"--- "): - filenames = True - else: - headscan = True - # -- ------------------------------------ - - # read out header - if headscan: - while not fe.is_empty and not fe.line.startswith(b"--- "): - header.append(fe.line) - fe.next() - if fe.is_empty: - if p is None: - debug("no patch data found") # error is shown later - self.errors += 1 - else: - info("%d unparsed bytes left at the end of stream" % len(b''.join(header))) - self.warnings += 1 - # TODO check for \No new line at the end.. - # TODO test for unparsed bytes - # otherwise error += 1 - # this is actually a loop exit - continue - - headscan = False - # switch to filenames state - filenames = True - - line = fe.line - lineno = fe.lineno - - - # hunkskip and hunkbody code skipped until definition of hunkhead is parsed - if hunkbody: - # [x] treat empty lines inside hunks as containing single space - # (this happens when diff is saved by copy/pasting to editor - # that strips trailing whitespace) - if line.strip(b"\r\n") == b"": - debug("expanding empty line in a middle of hunk body") - self.warnings += 1 - line = b' ' + line - - # process line first - if re.match(b"^[- \\+\\\\]", line): - # gather stats about line endings - if line.endswith(b"\r\n"): - p.hunkends["crlf"] += 1 - elif line.endswith(b"\n"): - p.hunkends["lf"] += 1 - elif line.endswith(b"\r"): - p.hunkends["cr"] += 1 - - if line.startswith(b"-"): - hunkactual["linessrc"] += 1 - elif line.startswith(b"+"): - hunkactual["linestgt"] += 1 - elif not line.startswith(b"\\"): - hunkactual["linessrc"] += 1 - hunkactual["linestgt"] += 1 - hunk.text.append(line) - # todo: handle \ No newline cases - else: - warning("invalid hunk no.%d at %d for target file %s" % (nexthunkno, lineno+1, p.target)) - # add hunk status node - hunk.invalid = True - p.hunks.append(hunk) - self.errors += 1 - # switch to hunkskip state - hunkbody = False - hunkskip = True - - # check exit conditions - if hunkactual["linessrc"] > hunk.linessrc or hunkactual["linestgt"] > hunk.linestgt: - warning("extra lines for hunk no.%d at %d for target %s" % (nexthunkno, lineno+1, p.target)) - # add hunk status node - hunk.invalid = True - p.hunks.append(hunk) - self.errors += 1 - # switch to hunkskip state - hunkbody = False - hunkskip = True - elif hunk.linessrc == hunkactual["linessrc"] and hunk.linestgt == hunkactual["linestgt"]: - # hunk parsed successfully - p.hunks.append(hunk) - # switch to hunkparsed state - hunkbody = False - hunkparsed = True - - # detect mixed window/unix line ends - ends = p.hunkends - if ((ends["cr"]!=0) + (ends["crlf"]!=0) + (ends["lf"]!=0)) > 1: - warning("inconsistent line ends in patch hunks for %s" % p.source) - self.warnings += 1 - if debugmode: - debuglines = dict(ends) - debuglines.update(file=p.target, hunk=nexthunkno) - debug("crlf: %(crlf)d lf: %(lf)d cr: %(cr)d\t - file: %(file)s hunk: %(hunk)d" % debuglines) - # fetch next line - continue - - if hunkskip: - if re_hunk_start.match(line): - # switch to hunkhead state - hunkskip = False - hunkhead = True - elif line.startswith(b"--- "): - # switch to filenames state - hunkskip = False - filenames = True - if debugmode and len(self.items) > 0: - debug("- %2d hunks for %s" % (len(p.hunks), p.source)) - - if filenames: - if line.startswith(b"--- "): - if srcname != None: - # XXX testcase - warning("skipping false patch for %s" % srcname) - srcname = None - # XXX header += srcname - # double source filename line is encountered - # attempt to restart from this second line - - # Files dated at Unix epoch don't exist, e.g.: - # '1970-01-01 01:00:00.000000000 +0100' - # They include timezone offsets. - # .. which can be parsed (if we remove the nanoseconds) - # .. by strptime() with: - # '%Y-%m-%d %H:%M:%S %z' - # .. but unfortunately this relies on the OSes libc - # strptime function and %z support is patchy, so we drop - # everything from the . onwards and group the year and time - # separately. - re_filename_date_time = b"^--- ([^\t]+)(?:\s([0-9-]+)\s([0-9:]+)|.*)" - match = re.match(re_filename_date_time, line) - # todo: support spaces in filenames - if match: - srcname = match.group(1).strip() - date = match.group(2) - time = match.group(3) - if (date == b'1970-01-01' or date == b'1969-12-31') and time.split(b':',1)[1] == b'00:00': - srcname = b'/dev/null' - else: - warning("skipping invalid filename at line %d" % (lineno+1)) - self.errors += 1 - # XXX p.header += line - # switch back to headscan state - filenames = False - headscan = True - elif not line.startswith(b"+++ "): - if srcname != None: - warning("skipping invalid patch with no target for %s" % srcname) - self.errors += 1 - srcname = None - # XXX header += srcname - # XXX header += line - else: - # this should be unreachable - warning("skipping invalid target patch") - filenames = False - headscan = True - else: - if tgtname != None: - # XXX seems to be a dead branch - warning("skipping invalid patch - double target at line %d" % (lineno+1)) - self.errors += 1 - srcname = None - tgtname = None - # XXX header += srcname - # XXX header += tgtname - # XXX header += line - # double target filename line is encountered - # switch back to headscan state - filenames = False - headscan = True - else: - re_filename_date_time = b"^\+\+\+ ([^\t]+)(?:\s([0-9-]+)\s([0-9:]+)|.*)" - match = re.match(re_filename_date_time, line) - if not match: - warning("skipping invalid patch - no target filename at line %d" % (lineno+1)) - self.errors += 1 - srcname = None - # switch back to headscan state - filenames = False - headscan = True - else: - tgtname = match.group(1).strip() - date = match.group(2) - time = match.group(3) - if (date == b'1970-01-01' or date == b'1969-12-31') and time.split(b':',1)[1] == b'00:00': - tgtname = b'/dev/null' - if p: # for the first run p is None - self.items.append(p) - p = Patch() - p.source = srcname - srcname = None - p.target = tgtname - tgtname = None - p.header = header - header = [] - # switch to hunkhead state - filenames = False - hunkhead = True - nexthunkno = 0 - p.hunkends = lineends.copy() - continue - - if hunkhead: - match = re.match(b"^@@ -(\d+)(,(\d+))? \+(\d+)(,(\d+))? @@(.*)", line) - if not match: - if not p.hunks: - warning("skipping invalid patch with no hunks for file %s" % p.source) - self.errors += 1 - # XXX review switch - # switch to headscan state - hunkhead = False - headscan = True - continue - else: - # TODO review condition case - # switch to headscan state - hunkhead = False - headscan = True - else: - hunk = Hunk() - hunk.startsrc = int(match.group(1)) - hunk.linessrc = 1 - if match.group(3): hunk.linessrc = int(match.group(3)) - hunk.starttgt = int(match.group(4)) - hunk.linestgt = 1 - if match.group(6): hunk.linestgt = int(match.group(6)) - hunk.invalid = False - hunk.desc = match.group(7)[1:].rstrip() - hunk.text = [] - - hunkactual["linessrc"] = hunkactual["linestgt"] = 0 - - # switch to hunkbody state - hunkhead = False - hunkbody = True - nexthunkno += 1 - continue - - # /while fe.next() - - if p: - self.items.append(p) - - if not hunkparsed: - if hunkskip: - warning("warning: finished with errors, some hunks may be invalid") - elif headscan: - if len(self.items) == 0: - warning("error: no patch data found!") - return False - else: # extra data at the end of file - pass - else: - warning("error: patch stream is incomplete!") - self.errors += 1 - if len(self.items) == 0: - return False - - if debugmode and len(self.items) > 0: - debug("- %2d hunks for %s" % (len(p.hunks), p.source)) - - # XXX fix total hunks calculation - debug("total files: %d total hunks: %d" % (len(self.items), - sum(len(p.hunks) for p in self.items))) - - # ---- detect patch and patchset types ---- - for idx, p in enumerate(self.items): - self.items[idx].type = self._detect_type(p) - - types = set([p.type for p in self.items]) - if len(types) > 1: - self.type = MIXED - else: - self.type = types.pop() - # -------- - - self._normalize_filenames() - - return (self.errors == 0) - - def _detect_type(self, p): - """ detect and return type for the specified Patch object - analyzes header and filenames info - - NOTE: must be run before filenames are normalized - """ - - # check for SVN - # - header starts with Index: - # - next line is ===... delimiter - # - filename is followed by revision number - # TODO add SVN revision - if (len(p.header) > 1 and p.header[-2].startswith(b"Index: ") - and p.header[-1].startswith(b"="*67)): - return SVN - - # common checks for both HG and GIT - DVCS = ((p.source.startswith(b'a/') or p.source == b'/dev/null') - and (p.target.startswith(b'b/') or p.target == b'/dev/null')) - - # GIT type check - # - header[-2] is like "diff --git a/oldname b/newname" - # - header[-1] is like "index .. " - # TODO add git rename diffs and add/remove diffs - # add git diff with spaced filename - # TODO http://www.kernel.org/pub/software/scm/git/docs/git-diff.html - - # Git patch header len is 2 min - if len(p.header) > 1: - # detect the start of diff header - there might be some comments before - for idx in reversed(range(len(p.header))): - if p.header[idx].startswith(b"diff --git"): - break - if p.header[idx].startswith(b'diff --git a/'): - if (idx+1 < len(p.header) - and re.match(b'(?:index \\w{7}..\\w{7} \\d{6}|new file mode \\d*)', p.header[idx+1])): - if DVCS: - return GIT - - # HG check - # - # - for plain HG format header is like "diff -r b2d9961ff1f5 filename" - # - for Git-style HG patches it is "diff --git a/oldname b/newname" - # - filename starts with a/, b/ or is equal to /dev/null - # - exported changesets also contain the header - # # HG changeset patch - # # User name@example.com - # ... - # TODO add MQ - # TODO add revision info - if len(p.header) > 0: - if DVCS and re.match(b'diff -r \\w{12} .*', p.header[-1]): - return HG - if DVCS and p.header[-1].startswith(b'diff --git a/'): - if len(p.header) == 1: # native Git patch header len is 2 - return HG - elif p.header[0].startswith(b'# HG changeset patch'): - return HG - - return PLAIN - - - def _normalize_filenames(self): - """ sanitize filenames, normalizing paths, i.e.: - 1. strip a/ and b/ prefixes from GIT and HG style patches - 2. remove all references to parent directories (with warning) - 3. translate any absolute paths to relative (with warning) - - [x] always use forward slashes to be crossplatform - (diff/patch were born as a unix utility after all) - - return None - """ - if debugmode: - debug("normalize filenames") - for i,p in enumerate(self.items): - if debugmode: - debug(" patch type = %s" % p.type) - debug(" source = %s" % p.source) - debug(" target = %s" % p.target) - if p.type in (HG, GIT): - debug("stripping a/ and b/ prefixes") - if p.source != b'/dev/null': - if not p.source.startswith(b"a/"): - warning("invalid source filename") - else: - p.source = p.source[2:] - if p.target != b'/dev/null': - if not p.target.startswith(b"b/"): - warning("invalid target filename") - else: - p.target = p.target[2:] - - p.source = xnormpath(p.source) - p.target = xnormpath(p.target) - - sep = b'/' # sep value can be hardcoded, but it looks nice this way - - # references to parent are not allowed - if p.source.startswith(b".." + sep): - warning("error: stripping parent path for source file patch no.%d" % (i+1)) - self.warnings += 1 - while p.source.startswith(b".." + sep): - p.source = p.source.partition(sep)[2] - if p.target.startswith(b".." + sep): - warning("error: stripping parent path for target file patch no.%d" % (i+1)) - self.warnings += 1 - while p.target.startswith(b".." + sep): - p.target = p.target.partition(sep)[2] - # absolute paths are not allowed - if (xisabs(p.source) and p.source != b'/dev/null') or \ - (xisabs(p.target) and p.target != b'/dev/null'): - warning("error: absolute paths are not allowed - file no.%d" % (i+1)) - self.warnings += 1 - if xisabs(p.source) and p.source != b'/dev/null': - warning("stripping absolute path from source name '%s'" % p.source) - p.source = xstrip(p.source) - if xisabs(p.target) and p.target != b'/dev/null': - warning("stripping absolute path from target name '%s'" % p.target) - p.target = xstrip(p.target) - - self.items[i].source = p.source - self.items[i].target = p.target - - - def diffstat(self): - """ calculate diffstat and return as a string - Notes: - - original diffstat ouputs target filename - - single + or - shouldn't escape histogram - """ - names = [] - insert = [] - delete = [] - delta = 0 # size change in bytes - namelen = 0 - maxdiff = 0 # max number of changes for single file - # (for histogram width calculation) - for patch in self.items: - i,d = 0,0 - for hunk in patch.hunks: - for line in hunk.text: - if line.startswith(b'+'): - i += 1 - delta += len(line)-1 - elif line.startswith(b'-'): - d += 1 - delta -= len(line)-1 - names.append(patch.target) - insert.append(i) - delete.append(d) - namelen = max(namelen, len(patch.target)) - maxdiff = max(maxdiff, i+d) - output = '' - statlen = len(str(maxdiff)) # stats column width - for i,n in enumerate(names): - # %-19s | %-4d %s - format = " %-" + str(namelen) + "s | %" + str(statlen) + "s %s\n" - - hist = '' - # -- calculating histogram -- - width = len(format % ('', '', '')) - histwidth = max(2, 80 - width) - if maxdiff < histwidth: - hist = "+"*insert[i] + "-"*delete[i] - else: - iratio = (float(insert[i]) / maxdiff) * histwidth - dratio = (float(delete[i]) / maxdiff) * histwidth - - # make sure every entry gets at least one + or - - iwidth = 1 if 0 < iratio < 1 else int(iratio) - dwidth = 1 if 0 < dratio < 1 else int(dratio) - #print(iratio, dratio, iwidth, dwidth, histwidth) - hist = "+"*int(iwidth) + "-"*int(dwidth) - # -- /calculating +- histogram -- - output += (format % (tostr(names[i]), str(insert[i] + delete[i]), hist)) - - output += (" %d files changed, %d insertions(+), %d deletions(-), %+d bytes" - % (len(names), sum(insert), sum(delete), delta)) - return output - - - def findfiles(self, old, new): - """ return tuple of source file, target file """ - if old == b'/dev/null': - handle, abspath = tempfile.mkstemp(suffix='pypatch') - abspath = abspath.encode() - # The source file must contain a line for the hunk matching to succeed. - os.write(handle, b' ') - os.close(handle) - if not exists(new): - handle = open(new, 'wb') - handle.close() - return abspath, new - elif exists(old): - return old, old - elif exists(new): - return new, new - elif new == b'/dev/null': - return None, None - else: - # [w] Google Code generates broken patches with its online editor - debug("broken patch from Google Code, stripping prefixes..") - if old.startswith(b'a/') and new.startswith(b'b/'): - old, new = old[2:], new[2:] - debug(" %s" % old) - debug(" %s" % new) - if exists(old): - return old, old - elif exists(new): - return new, new - return None, None - - def _strip_prefix(self, filename): - if filename.startswith(b'a/') or filename.startswith(b'b/'): - return filename[2:] - return filename - - def decode_clean(self, path, prefix): - path = path.decode("utf-8").replace("\\", "/") - if path.startswith(prefix): - path = path[2:] - return path - - def strip_path(self, path, base_path, strip=0): - tokens = path.split("/") - if len(tokens) > 1: - tokens = tokens[strip:] - path = "/".join(tokens) - if base_path: - path = os.path.join(base_path, path) - return path - # account for new and deleted files, upstream dep won't fix them - - - - - def apply(self, strip=0, root=None, fuzz=False): - """ Apply parsed patch, optionally stripping leading components - from file paths. `root` parameter specifies working dir. - :param strip: Strip patch path - :param root: Folder to apply the patch - :param fuzz: Accept fuzzy patches - return True on success - """ - items = [] - for item in self.items: - source = self.decode_clean(item.source, "a/") - target = self.decode_clean(item.target, "b/") - if "dev/null" in source: - target = self.strip_path(target, root, strip) - hunks = [s.decode("utf-8") for s in item.hunks[0].text] - new_file = "".join(hunk[1:] for hunk in hunks) - save(target, new_file) - elif "dev/null" in target: - source = self.strip_path(source, root, strip) - safe_unlink(source) - else: - items.append(item) - self.items = items - - if root: - prevdir = os.getcwd() - os.chdir(root) - - total = len(self.items) - errors = 0 - if strip: - # [ ] test strip level exceeds nesting level - # [ ] test the same only for selected files - # [ ] test if files end up being on the same level - try: - strip = int(strip) - except ValueError: - errors += 1 - warning("error: strip parameter '%s' must be an integer" % strip) - strip = 0 - - #for fileno, filename in enumerate(self.source): - for i,p in enumerate(self.items): - if strip: - debug("stripping %s leading component(s) from:" % strip) - debug(" %s" % p.source) - debug(" %s" % p.target) - old = p.source if p.source == b'/dev/null' else pathstrip(p.source, strip) - new = p.target if p.target == b'/dev/null' else pathstrip(p.target, strip) - else: - old, new = p.source, p.target - - filenameo, filenamen = self.findfiles(old, new) - - if not filenameo or not filenamen: - error("source/target file does not exist:\n --- %s\n +++ %s" % (old, new)) - errors += 1 - continue - if not isfile(filenameo): - error("not a file - %s" % filenameo) - errors += 1 - continue - - # [ ] check absolute paths security here - debug("processing %d/%d:\t %s" % (i+1, total, filenamen)) - - # validate before patching - f2fp = open(filenameo, 'rb') - hunkno = 0 - hunk = p.hunks[hunkno] - hunkfind = [] - hunkreplace = [] - validhunks = 0 - canpatch = False - for lineno, line in enumerate(f2fp): - if lineno+1 < hunk.startsrc: - continue - elif lineno+1 == hunk.startsrc: - hunkfind = [x[1:].rstrip(b"\r\n") for x in hunk.text if x[0] in b" -"] - hunkreplace = [x[1:].rstrip(b"\r\n") for x in hunk.text if x[0] in b" +"] - #pprint(hunkreplace) - hunklineno = 0 - - # todo \ No newline at end of file - - # check hunks in source file - if lineno+1 < hunk.startsrc+len(hunkfind): - if line.rstrip(b"\r\n") == hunkfind[hunklineno]: - hunklineno += 1 - else: - warning("file %d/%d:\t %s" % (i+1, total, filenamen)) - warning(" hunk no.%d doesn't match source file at line %d" % (hunkno+1, lineno+1)) - warning(" expected: %s" % hunkfind[hunklineno]) - warning(" actual : %s" % line.rstrip(b"\r\n")) - if fuzz: - hunklineno += 1 - else: - # not counting this as error, because file may already be patched. - # check if file is already patched is done after the number of - # invalid hunks if found - # TODO: check hunks against source/target file in one pass - # API - check(stream, srchunks, tgthunks) - # return tuple (srcerrs, tgterrs) - - # continue to check other hunks for completeness - hunkno += 1 - if hunkno < len(p.hunks): - hunk = p.hunks[hunkno] - continue - else: - break - - # check if processed line is the last line - if len(hunkfind) == 0 or lineno+1 == hunk.startsrc+len(hunkfind)-1: - debug(" hunk no.%d for file %s -- is ready to be patched" % (hunkno+1, filenamen)) - hunkno+=1 - validhunks+=1 - if hunkno < len(p.hunks): - hunk = p.hunks[hunkno] - else: - if validhunks == len(p.hunks): - # patch file - canpatch = True - break - else: - if hunkno < len(p.hunks): - error("premature end of source file %s at hunk %d" % (filenameo, hunkno+1)) - errors += 1 - - f2fp.close() - - if validhunks < len(p.hunks): - if self._match_file_hunks(filenameo, p.hunks): - warning("already patched %s" % filenameo) - else: - if fuzz: - warning("source file is different - %s" % filenameo) - else: - error("source file is different - %s" % filenameo) - errors += 1 - if canpatch: - backupname = filenamen+b".orig" - if exists(backupname): - warning("can't backup original file to %s - aborting" % backupname) - errors += 1 - else: - shutil.move(filenamen, backupname) - if self.write_hunks(backupname if filenameo == filenamen else filenameo, filenamen, p.hunks): - info("successfully patched %d/%d:\t %s" % (i+1, total, filenamen)) - safe_unlink(backupname) - if new == b'/dev/null': - # check that filename is of size 0 and delete it. - if os.path.getsize(filenamen) > 0: - warning("expected patched file to be empty as it's marked as deletion:\t %s" % filenamen) - safe_unlink(filenamen) - else: - errors += 1 - warning("error patching file %s" % filenamen) - shutil.copy(filenamen, filenamen+".invalid") - warning("invalid version is saved to %s" % filenamen+".invalid") - # todo: proper rejects - shutil.move(backupname, filenamen) - - if root: - os.chdir(prevdir) - - # todo: check for premature eof - return (errors == 0) - - - def _reverse(self): - """ reverse patch direction (this doesn't touch filenames) """ - for p in self.items: - for h in p.hunks: - h.startsrc, h.starttgt = h.starttgt, h.startsrc - h.linessrc, h.linestgt = h.linestgt, h.linessrc - for i,line in enumerate(h.text): - # need to use line[0:1] here, because line[0] - # returns int instead of bytes on Python 3 - if line[0:1] == b'+': - h.text[i] = b'-' + line[1:] - elif line[0:1] == b'-': - h.text[i] = b'+' +line[1:] - - def revert(self, strip=0, root=None): - """ apply patch in reverse order """ - reverted = copy.deepcopy(self) - reverted._reverse() - return reverted.apply(strip, root) - - - def can_patch(self, filename): - """ Check if specified filename can be patched. Returns None if file can - not be found among source filenames. False if patch can not be applied - clearly. True otherwise. - - :returns: True, False or None - """ - filename = abspath(filename) - for p in self.items: - if filename == abspath(p.source): - return self._match_file_hunks(filename, p.hunks) - return None - - - def _match_file_hunks(self, filepath, hunks): - matched = True - fp = open(abspath(filepath), 'rb') - - class NoMatch(Exception): - pass - - lineno = 1 - line = fp.readline() - try: - for hno, h in enumerate(hunks): - # skip to first line of the hunk - while lineno < h.starttgt: - if not len(line): # eof - debug("check failed - premature eof before hunk: %d" % (hno+1)) - raise NoMatch - line = fp.readline() - lineno += 1 - for hline in h.text: - if hline.startswith(b"-"): - continue - if not len(line): - debug("check failed - premature eof on hunk: %d" % (hno+1)) - # todo: \ No newline at the end of file - raise NoMatch - if line.rstrip(b"\r\n") != hline[1:].rstrip(b"\r\n"): - debug("file is not patched - failed hunk: %d" % (hno+1)) - raise NoMatch - line = fp.readline() - lineno += 1 - - except NoMatch: - matched = False - # todo: display failed hunk, i.e. expected/found - - fp.close() - return matched - - - def patch_stream(self, instream, hunks): - """ Generator that yields stream patched with hunks iterable - - Converts lineends in hunk lines to the best suitable format - autodetected from input - """ - - # todo: At the moment substituted lineends may not be the same - # at the start and at the end of patching. Also issue a - # warning/throw about mixed lineends (is it really needed?) - - hunks = iter(hunks) - - srclineno = 1 - - lineends = {b'\n':0, b'\r\n':0, b'\r':0} - def get_line(): - """ - local utility function - return line from source stream - collecting line end statistics on the way - """ - line = instream.readline() - # 'U' mode works only with text files - if line.endswith(b"\r\n"): - lineends[b"\r\n"] += 1 - elif line.endswith(b"\n"): - lineends[b"\n"] += 1 - elif line.endswith(b"\r"): - lineends[b"\r"] += 1 - return line - - for hno, h in enumerate(hunks): - debug("hunk %d" % (hno+1)) - # skip to line just before hunk starts - while srclineno < h.startsrc: - yield get_line() - srclineno += 1 - - for hline in h.text: - # todo: check \ No newline at the end of file - if hline.startswith(b"-") or hline.startswith(b"\\"): - get_line() - srclineno += 1 - continue - else: - if not hline.startswith(b"+"): - yield get_line() - srclineno += 1 - continue - line2write = hline[1:] - # detect if line ends are consistent in source file - if sum([bool(lineends[x]) for x in lineends]) == 1: - newline = [x for x in lineends if lineends[x] != 0][0] - yield line2write.rstrip(b"\r\n")+newline - else: # newlines are mixed - yield line2write - - for line in instream: - yield line - - - def write_hunks(self, srcname, tgtname, hunks): - src = open(srcname, "rb") - tgt = open(tgtname, "wb") - - debug("processing target file %s" % tgtname) - - tgt.writelines(self.patch_stream(src, hunks)) - - tgt.close() - src.close() - # [ ] TODO: add test for permission copy - shutil.copymode(srcname, tgtname) - return True - - - def dump(self): - for p in self.items: - for headline in p.header: - print(headline.rstrip('\n')) - print('--- ' + p.source) - print('+++ ' + p.target) - for h in p.hunks: - print('@@ -%s,%s +%s,%s @@' % (h.startsrc, h.linessrc, h.starttgt, h.linestgt)) - for line in h.text: - print(line.rstrip('\n')) - - -def main(): - from optparse import OptionParser - from os.path import exists - import sys - - opt = OptionParser(usage="1. %prog [options] unified.diff\n" - " 2. %prog [options] http://host/patch\n" - " 3. %prog [options] -- < unified.diff", - version="python-patch %s" % __version__) - opt.add_option("-q", "--quiet", action="store_const", dest="verbosity", - const=0, help="print only warnings and errors", default=1) - opt.add_option("-v", "--verbose", action="store_const", dest="verbosity", - const=2, help="be verbose") - opt.add_option("--debug", action="store_true", dest="debugmode", help="debug mode") - opt.add_option("--diffstat", action="store_true", dest="diffstat", - help="print diffstat and exit") - opt.add_option("-d", "--directory", metavar='DIR', - help="specify root directory for applying patch") - opt.add_option("-p", "--strip", type="int", metavar='N', default=0, - help="strip N path components from filenames") - opt.add_option("--revert", action="store_true", - help="apply patch in reverse order (unpatch)") - opt.add_option("-f", "--fuzz", action="store_true", dest="fuzz", help="Accept fuuzzy patches") - (options, args) = opt.parse_args() - - if not args and sys.argv[-1:] != ['--']: - opt.print_version() - opt.print_help() - sys.exit() - readstdin = (sys.argv[-1:] == ['--'] and not args) - - verbosity_levels = {0:logging.WARNING, 1:logging.INFO, 2:logging.DEBUG} - loglevel = verbosity_levels[options.verbosity] - logformat = "%(message)s" - logger.setLevel(loglevel) - streamhandler.setFormatter(logging.Formatter(logformat)) - - if options.debugmode: - setdebug() # this sets global debugmode variable - - if readstdin: - patch = PatchSet(sys.stdin) - else: - patchfile = args[0] - urltest = patchfile.split(':')[0] - if (':' in patchfile and urltest.isalpha() - and len(urltest) > 1): # one char before : is a windows drive letter - patch = fromurl(patchfile) - else: - if not exists(patchfile) or not isfile(patchfile): - sys.exit("patch file does not exist - %s" % patchfile) - patch = fromfile(patchfile) - - if options.diffstat: - print(patch.diffstat()) - sys.exit(0) - - if not patch: - error("Could not parse patch") - sys.exit(-1) - - #pprint(patch) - if options.revert: - patch.revert(options.strip, root=options.directory) or sys.exit(-1) - else: - patch.apply(options.strip, root=options.directory, fuzz=options.fuzz) or sys.exit(-1) - - # todo: document and test line ends handling logic - patch_ng.py detects proper line-endings - # for inserted hunks and issues a warning if patched file has incosistent line ends - - -if __name__ == "__main__": - main() - -# Legend: -# [ ] - some thing to be done -# [w] - official wart, external or internal that is unlikely to be fixed - -# [ ] API break (2.x) wishlist -# PatchSet.items --> PatchSet.patches - -# [ ] run --revert test for all dataset items -# [ ] run .parse() / .dump() test for dataset diff --git a/patches/0001-RadioLib-SPItransfer-virtual.patch b/patches/0001-RadioLib-SPItransfer-virtual.patch deleted file mode 100644 index 07eb31f77..000000000 --- a/patches/0001-RadioLib-SPItransfer-virtual.patch +++ /dev/null @@ -1,15 +0,0 @@ -index 3a7b098..aa38f6d 100644 ---- a/src/Module.h -+++ b/src/Module.h -@@ -361,9 +361,9 @@ class Module { - // helper functions to set up SPI overrides on Arduino - #if defined(RADIOLIB_BUILD_ARDUINO) - void SPIbegin(); -- void SPIbeginTransaction(); -+ virtual void SPIbeginTransaction(); - uint8_t SPItransfer(uint8_t b); -- void SPIendTransaction(); -+ virtual void SPIendTransaction(); - void SPIend(); - #endif - diff --git a/platformio.ini b/platformio.ini index 166745415..60e9fe68e 100644 --- a/platformio.ini +++ b/platformio.ini @@ -61,7 +61,8 @@ framework = arduino lib_deps = ${env.lib_deps} ; Portduino is using meshtastic fork for now - https://github.com/jgromes/RadioLib.git#3df3b092ebf412bd0b26524e7b296733bd6a62f7 + https://github.com/jgromes/RadioLib.git + build_flags = ${env.build_flags} -Os # -DRADIOLIB_GODMODE build_src_filter = ${env.build_src_filter} - @@ -94,9 +95,6 @@ build_src_filter = ${arduino_base.build_src_filter} - upload_speed = 921600 debug_init_break = tbreak setup -extra_scripts = - ${env.extra_scripts} - pre:bin/apply_patches.py # Remove -DMYNEWT_VAL_BLE_HS_LOG_LVL=LOG_LEVEL_CRITICAL for low level BLE logging. # See library directory for BLE logging possible values: .pio/libdeps/tbeam/NimBLE-Arduino/src/log_common/log_common.h @@ -149,9 +147,6 @@ build_src_filter = ${arduino_base.build_src_filter} - - - - - - lib_ignore = BluetoothOTA -extra_scripts = - ${env.extra_scripts} - pre:bin/apply_patches.py [nrf52840_base] extends = nrf52_base From 4a6cad6e46ce0ef3690898c82b0be68060be5841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Mon, 13 Jun 2022 23:00:50 +0200 Subject: [PATCH 11/20] Set TX Power to some meaningful value --- src/mesh/RadioInterface.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/mesh/RadioInterface.cpp b/src/mesh/RadioInterface.cpp index 635862956..12de75086 100644 --- a/src/mesh/RadioInterface.cpp +++ b/src/mesh/RadioInterface.cpp @@ -402,6 +402,12 @@ void RadioInterface::applyModemConfig() power = loraConfig.tx_power; assert(myRegion); // Should have been found in init + if ((power == 0) || (power > myRegion->powerLimit)) + power = myRegion->powerLimit; + + if (power == 0) + power = 17; // Default to default power if we don't have a valid power + // Calculate the number of channels uint32_t numChannels = floor((myRegion->freqEnd - myRegion->freqStart) / (myRegion->spacing + (bw / 1000))); From 058b5ceddd98099145c7f9d88fc361bb932e466e Mon Sep 17 00:00:00 2001 From: caveman99 Date: Wed, 15 Jun 2022 14:44:33 +0000 Subject: [PATCH 12/20] [create-pull-request] automated change --- protobufs | 2 +- src/mesh/generated/deviceonly.pb.h | 14 +++++-- src/mesh/generated/localonly.pb.h | 54 ++++++++++++++++----------- src/mesh/generated/module_config.pb.h | 2 +- 4 files changed, 45 insertions(+), 27 deletions(-) diff --git a/protobufs b/protobufs index 8f616d3ee..9b0148e86 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 8f616d3eefd4edc86b915f3d7fae827847bdcd95 +Subproject commit 9b0148e86b28d995c56e3aca209f5e2af6360c5b diff --git a/src/mesh/generated/deviceonly.pb.h b/src/mesh/generated/deviceonly.pb.h index 6da17270e..dc7a305db 100644 --- a/src/mesh/generated/deviceonly.pb.h +++ b/src/mesh/generated/deviceonly.pb.h @@ -28,6 +28,10 @@ typedef struct _ChannelFile { /* The channels our node knows about */ pb_size_t channels_count; Channel channels[8]; + /* A version integer used to invalidate old save files when we make + incompatible changes This integer is set at build time and is private to + NodeDB.cpp in the device code. */ + uint32_t version; } ChannelFile; /* This message is never sent over the wire, but it is used for serializing DB @@ -93,14 +97,15 @@ extern "C" { /* Initializer values for message structs */ #define DeviceState_init_default {false, MyNodeInfo_init_default, false, User_init_default, 0, {NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default, NodeInfo_init_default}, 0, {MeshPacket_init_default}, false, MeshPacket_init_default, 0, 0, 0} -#define ChannelFile_init_default {0, {Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default}} +#define ChannelFile_init_default {0, {Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default, Channel_init_default}, 0} #define OEMStore_init_default {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} #define DeviceState_init_zero {false, MyNodeInfo_init_zero, false, User_init_zero, 0, {NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero, NodeInfo_init_zero}, 0, {MeshPacket_init_zero}, false, MeshPacket_init_zero, 0, 0, 0} -#define ChannelFile_init_zero {0, {Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero}} +#define ChannelFile_init_zero {0, {Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero, Channel_init_zero}, 0} #define OEMStore_init_zero {0, 0, {0, {0}}, _ScreenFonts_MIN, ""} /* Field tags (for use in manual encoding/decoding) */ #define ChannelFile_channels_tag 1 +#define ChannelFile_version_tag 2 #define DeviceState_my_node_tag 2 #define DeviceState_owner_tag 3 #define DeviceState_node_db_tag 4 @@ -134,7 +139,8 @@ X(a, STATIC, SINGULAR, BOOL, did_gps_reset, 11) #define DeviceState_rx_text_message_MSGTYPE MeshPacket #define ChannelFile_FIELDLIST(X, a) \ -X(a, STATIC, REPEATED, MESSAGE, channels, 1) +X(a, STATIC, REPEATED, MESSAGE, channels, 1) \ +X(a, STATIC, SINGULAR, UINT32, version, 2) #define ChannelFile_CALLBACK NULL #define ChannelFile_DEFAULT NULL #define ChannelFile_channels_MSGTYPE Channel @@ -158,7 +164,7 @@ extern const pb_msgdesc_t OEMStore_msg; #define OEMStore_fields &OEMStore_msg /* Maximum encoded size of messages (where known) */ -#define ChannelFile_size 624 +#define ChannelFile_size 630 #define DeviceState_size 23728 #define OEMStore_size 2106 diff --git a/src/mesh/generated/localonly.pb.h b/src/mesh/generated/localonly.pb.h index 06922a88d..60c8ed2d5 100644 --- a/src/mesh/generated/localonly.pb.h +++ b/src/mesh/generated/localonly.pb.h @@ -13,48 +13,56 @@ /* Struct definitions */ typedef struct _LocalConfig { - /* TODO: REPLACE */ + /* The part of the config that is specific to the Device */ bool has_device; Config_DeviceConfig device; - /* TODO: REPLACE */ + /* The part of the config that is specific to the GPS Position */ bool has_position; Config_PositionConfig position; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Power settings */ bool has_power; Config_PowerConfig power; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Wifi Settings */ bool has_wifi; Config_WiFiConfig wifi; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Display */ bool has_display; Config_DisplayConfig display; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Lora Radio */ bool has_lora; Config_LoRaConfig lora; + /* A version integer used to invalidate old save files when we make + incompatible changes This integer is set at build time and is private to + NodeDB.cpp in the device code. */ + uint32_t version; } LocalConfig; typedef struct _LocalModuleConfig { - /* TODO: REPLACE */ + /* The part of the config that is specific to the MQTT module */ bool has_mqtt; ModuleConfig_MQTTConfig mqtt; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Serial module */ bool has_serial; ModuleConfig_SerialConfig serial; - /* TODO: REPLACE */ + /* The part of the config that is specific to the ExternalNotification module */ bool has_external_notification; ModuleConfig_ExternalNotificationConfig external_notification; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Store & Forward module */ bool has_store_forward; ModuleConfig_StoreForwardConfig store_forward; - /* TODO: REPLACE */ + /* The part of the config that is specific to the RangeTest module */ bool has_range_test; ModuleConfig_RangeTestConfig range_test; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Telemetry module */ bool has_telemetry; ModuleConfig_TelemetryConfig telemetry; - /* TODO: REPLACE */ + /* The part of the config that is specific to the Canned Message module */ bool has_canned_message; ModuleConfig_CannedMessageConfig canned_message; + /* A version integer used to invalidate old save files when we make + incompatible changes This integer is set at build time and is private to + NodeDB.cpp in the device code. */ + uint32_t version; } LocalModuleConfig; @@ -63,10 +71,10 @@ extern "C" { #endif /* Initializer values for message structs */ -#define LocalConfig_init_default {false, Config_DeviceConfig_init_default, false, Config_PositionConfig_init_default, false, Config_PowerConfig_init_default, false, Config_WiFiConfig_init_default, false, Config_DisplayConfig_init_default, false, Config_LoRaConfig_init_default} -#define LocalModuleConfig_init_default {false, ModuleConfig_MQTTConfig_init_default, false, ModuleConfig_SerialConfig_init_default, false, ModuleConfig_ExternalNotificationConfig_init_default, false, ModuleConfig_StoreForwardConfig_init_default, false, ModuleConfig_RangeTestConfig_init_default, false, ModuleConfig_TelemetryConfig_init_default, false, ModuleConfig_CannedMessageConfig_init_default} -#define LocalConfig_init_zero {false, Config_DeviceConfig_init_zero, false, Config_PositionConfig_init_zero, false, Config_PowerConfig_init_zero, false, Config_WiFiConfig_init_zero, false, Config_DisplayConfig_init_zero, false, Config_LoRaConfig_init_zero} -#define LocalModuleConfig_init_zero {false, ModuleConfig_MQTTConfig_init_zero, false, ModuleConfig_SerialConfig_init_zero, false, ModuleConfig_ExternalNotificationConfig_init_zero, false, ModuleConfig_StoreForwardConfig_init_zero, false, ModuleConfig_RangeTestConfig_init_zero, false, ModuleConfig_TelemetryConfig_init_zero, false, ModuleConfig_CannedMessageConfig_init_zero} +#define LocalConfig_init_default {false, Config_DeviceConfig_init_default, false, Config_PositionConfig_init_default, false, Config_PowerConfig_init_default, false, Config_WiFiConfig_init_default, false, Config_DisplayConfig_init_default, false, Config_LoRaConfig_init_default, 0} +#define LocalModuleConfig_init_default {false, ModuleConfig_MQTTConfig_init_default, false, ModuleConfig_SerialConfig_init_default, false, ModuleConfig_ExternalNotificationConfig_init_default, false, ModuleConfig_StoreForwardConfig_init_default, false, ModuleConfig_RangeTestConfig_init_default, false, ModuleConfig_TelemetryConfig_init_default, false, ModuleConfig_CannedMessageConfig_init_default, 0} +#define LocalConfig_init_zero {false, Config_DeviceConfig_init_zero, false, Config_PositionConfig_init_zero, false, Config_PowerConfig_init_zero, false, Config_WiFiConfig_init_zero, false, Config_DisplayConfig_init_zero, false, Config_LoRaConfig_init_zero, 0} +#define LocalModuleConfig_init_zero {false, ModuleConfig_MQTTConfig_init_zero, false, ModuleConfig_SerialConfig_init_zero, false, ModuleConfig_ExternalNotificationConfig_init_zero, false, ModuleConfig_StoreForwardConfig_init_zero, false, ModuleConfig_RangeTestConfig_init_zero, false, ModuleConfig_TelemetryConfig_init_zero, false, ModuleConfig_CannedMessageConfig_init_zero, 0} /* Field tags (for use in manual encoding/decoding) */ #define LocalConfig_device_tag 1 @@ -75,6 +83,7 @@ extern "C" { #define LocalConfig_wifi_tag 4 #define LocalConfig_display_tag 5 #define LocalConfig_lora_tag 6 +#define LocalConfig_version_tag 7 #define LocalModuleConfig_mqtt_tag 1 #define LocalModuleConfig_serial_tag 2 #define LocalModuleConfig_external_notification_tag 3 @@ -82,6 +91,7 @@ extern "C" { #define LocalModuleConfig_range_test_tag 5 #define LocalModuleConfig_telemetry_tag 6 #define LocalModuleConfig_canned_message_tag 7 +#define LocalModuleConfig_version_tag 8 /* Struct field encoding specification for nanopb */ #define LocalConfig_FIELDLIST(X, a) \ @@ -90,7 +100,8 @@ X(a, STATIC, OPTIONAL, MESSAGE, position, 2) \ X(a, STATIC, OPTIONAL, MESSAGE, power, 3) \ X(a, STATIC, OPTIONAL, MESSAGE, wifi, 4) \ X(a, STATIC, OPTIONAL, MESSAGE, display, 5) \ -X(a, STATIC, OPTIONAL, MESSAGE, lora, 6) +X(a, STATIC, OPTIONAL, MESSAGE, lora, 6) \ +X(a, STATIC, SINGULAR, UINT32, version, 7) #define LocalConfig_CALLBACK NULL #define LocalConfig_DEFAULT NULL #define LocalConfig_device_MSGTYPE Config_DeviceConfig @@ -107,7 +118,8 @@ X(a, STATIC, OPTIONAL, MESSAGE, external_notification, 3) \ X(a, STATIC, OPTIONAL, MESSAGE, store_forward, 4) \ X(a, STATIC, OPTIONAL, MESSAGE, range_test, 5) \ X(a, STATIC, OPTIONAL, MESSAGE, telemetry, 6) \ -X(a, STATIC, OPTIONAL, MESSAGE, canned_message, 7) +X(a, STATIC, OPTIONAL, MESSAGE, canned_message, 7) \ +X(a, STATIC, SINGULAR, UINT32, version, 8) #define LocalModuleConfig_CALLBACK NULL #define LocalModuleConfig_DEFAULT NULL #define LocalModuleConfig_mqtt_MSGTYPE ModuleConfig_MQTTConfig @@ -126,8 +138,8 @@ extern const pb_msgdesc_t LocalModuleConfig_msg; #define LocalModuleConfig_fields &LocalModuleConfig_msg /* Maximum encoded size of messages (where known) */ -#define LocalConfig_size 315 -#define LocalModuleConfig_size 282 +#define LocalConfig_size 321 +#define LocalModuleConfig_size 288 #ifdef __cplusplus } /* extern "C" */ diff --git a/src/mesh/generated/module_config.pb.h b/src/mesh/generated/module_config.pb.h index f68b1655c..56d1b3b26 100644 --- a/src/mesh/generated/module_config.pb.h +++ b/src/mesh/generated/module_config.pb.h @@ -115,7 +115,7 @@ typedef struct _ModuleConfig_TelemetryConfig { uint32_t environment_sensor_pin; } ModuleConfig_TelemetryConfig; -/* TODO: REPLACE */ +/* Module Config */ typedef struct _ModuleConfig { /* TODO: REPLACE */ pb_size_t which_payloadVariant; From d18aa2e7cbcf39f0e302928c55aaae4d9330956c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 15 Jun 2022 16:52:04 +0200 Subject: [PATCH 13/20] add file version to local savefiles --- src/mesh/NodeDB.cpp | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index a9ee9df1e..d4d5218ec 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -48,7 +48,7 @@ DeviceState versions used to be defined in the .proto file but really only this #define here. */ -#define DEVICESTATE_CUR_VER 11 +#define DEVICESTATE_CUR_VER 13 #define DEVICESTATE_MIN_VER DEVICESTATE_CUR_VER // FIXME - move this somewhere else @@ -146,6 +146,7 @@ bool NodeDB::resetRadioConfig() void NodeDB::installDefaultConfig() { memset(&config, 0, sizeof(LocalConfig)); + config.version = DEVICESTATE_CUR_VER; config.has_device = true; config.has_display = true; config.has_lora = true; @@ -165,6 +166,7 @@ void NodeDB::installDefaultConfig() void NodeDB::installDefaultModuleConfig() { memset(&moduleConfig, 0, sizeof(ModuleConfig)); + moduleConfig.version = DEVICESTATE_CUR_VER; moduleConfig.has_canned_message = true; moduleConfig.has_external_notification = true; moduleConfig.has_mqtt = true; @@ -189,6 +191,7 @@ void NodeDB::installDefaultModuleConfig() void NodeDB::installDefaultChannels() { memset(&channelFile, 0, sizeof(ChannelFile)); + channelFile.version = DEVICESTATE_CUR_VER; } void NodeDB::installDefaultDeviceState() @@ -347,20 +350,41 @@ void NodeDB::loadFromDisk() DEBUG_MSG("Warn: devicestate %d is old, discarding\n", devicestate.version); installDefaultDeviceState(); } else { - DEBUG_MSG("Loaded saved preferences version %d\n", devicestate.version); + DEBUG_MSG("Loaded saved devicestate version %d\n", devicestate.version); } } if (!loadProto(configfile, LocalConfig_size, sizeof(LocalConfig), LocalConfig_fields, &config)) { installDefaultConfig(); // Our in RAM copy might now be corrupt + } else { + if (config.version < DEVICESTATE_MIN_VER) { + DEBUG_MSG("Warn: config %d is old, discarding\n", config.version); + installDefaultConfig(); + } else { + DEBUG_MSG("Loaded saved config version %d\n", config.version); + } } if (!loadProto(moduleConfigfile, LocalModuleConfig_size, sizeof(LocalModuleConfig), LocalModuleConfig_fields, &moduleConfig)) { installDefaultModuleConfig(); // Our in RAM copy might now be corrupt + } else { + if (moduleConfig.version < DEVICESTATE_MIN_VER) { + DEBUG_MSG("Warn: moduleConfig %d is old, discarding\n", moduleConfig.version); + installDefaultModuleConfig(); + } else { + DEBUG_MSG("Loaded saved moduleConfig version %d\n", moduleConfig.version); + } } if (!loadProto(channelfile, ChannelFile_size, sizeof(ChannelFile), ChannelFile_fields, &channelFile)) { installDefaultChannels(); // Our in RAM copy might now be corrupt + } else { + if (channelFile.version < DEVICESTATE_MIN_VER) { + DEBUG_MSG("Warn: channelFile %d is old, discarding\n", channelFile.version); + installDefaultChannels(); + } else { + DEBUG_MSG("Loaded saved channelFile version %d\n", channelFile.version); + } } } From b1274799610ddf81e036ed11392d0158839b12b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 15 Jun 2022 17:09:42 +0200 Subject: [PATCH 14/20] - Refactored factory reset a bit to not installDefaultDeviceState twice on ESP32 - clear BLE bonds on settings version increase --- src/mesh/NodeDB.cpp | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index d4d5218ec..c4ab8cad0 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -90,18 +90,17 @@ bool NodeDB::resetRadioConfig() // radioConfig.has_preferences = true; if (config.device.factory_reset) { DEBUG_MSG("Performing factory reset!\n"); - installDefaultDeviceState(); -#ifndef NO_ESP32 - // This will erase what's in NVS including ssl keys, persistant variables and ble pairing - nvs_flash_erase(); -#endif -#ifdef NRF52_SERIES // first, remove the "/prefs" (this removes most prefs) FSCom.rmdir_r("/prefs"); // second, install default state (this will deal with the duplicate mac address issue) installDefaultDeviceState(); // third, write to disk saveToDisk(); +#ifndef NO_ESP32 + // This will erase what's in NVS including ssl keys, persistant variables and ble pairing + nvs_flash_erase(); +#endif +#ifdef NRF52_SERIES Bluefruit.begin(); DEBUG_MSG("Clearing bluetooth bonds!\n"); bond_print_list(BLE_GAP_ROLE_PERIPH); @@ -176,18 +175,6 @@ void NodeDB::installDefaultModuleConfig() moduleConfig.has_telemetry = true; } -// void NodeDB::installDefaultRadioConfig() -// { -// memset(&radioConfig, 0, sizeof(radioConfig)); -// radioConfig.has_preferences = true; -// resetRadioConfig(); - -// // for backward compat, default position flags are BAT+ALT+MSL (0x23 = 35) -// config.position.position_flags = -// (Config_PositionConfig_PositionFlags_POS_BATTERY | Config_PositionConfig_PositionFlags_POS_ALTITUDE | -// Config_PositionConfig_PositionFlags_POS_ALT_MSL); -// } - void NodeDB::installDefaultChannels() { memset(&channelFile, 0, sizeof(ChannelFile)); @@ -349,6 +336,18 @@ void NodeDB::loadFromDisk() if (devicestate.version < DEVICESTATE_MIN_VER) { DEBUG_MSG("Warn: devicestate %d is old, discarding\n", devicestate.version); installDefaultDeviceState(); +#ifndef NO_ESP32 + // This will erase what's in NVS including ssl keys, persistant variables and ble pairing + nvs_flash_erase(); +#endif +#ifdef NRF52_SERIES + Bluefruit.begin(); + DEBUG_MSG("Clearing bluetooth bonds!\n"); + bond_print_list(BLE_GAP_ROLE_PERIPH); + bond_print_list(BLE_GAP_ROLE_CENTRAL); + Bluefruit.Periph.clearBonds(); + Bluefruit.Central.clearBonds(); +#endif } else { DEBUG_MSG("Loaded saved devicestate version %d\n", devicestate.version); } From 125f76d9845f736099d08727171cbb3a4a2fa409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Wed, 15 Jun 2022 17:52:37 +0200 Subject: [PATCH 15/20] Don't use rmdir_r but roll our own version. --- src/FSCommon.cpp | 28 ++++++++++++++++++++++++++++ src/FSCommon.h | 4 +++- src/mesh/NodeDB.cpp | 2 +- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/FSCommon.cpp b/src/FSCommon.cpp index 3a2f8a92e..5293c4015 100644 --- a/src/FSCommon.cpp +++ b/src/FSCommon.cpp @@ -28,6 +28,34 @@ void listDir(const char * dirname, uint8_t levels) #endif } +void rmDir(const char * dirname) +#ifdef FSCom +{ + File root = FSCom.open(dirname); + if(!root){ + return; + } + if(!root.isDirectory()){ + return; + } + + File file = root.openNextFile(); + while(file){ + if(file.isDirectory() && !String(file.name()).endsWith(".")) { + file.close(); + rmDir(file.name()); + FSCom.rmdir(file.name()); + } else { + file.close(); + FSCom.remove(file.name()); + } + file.close(); + file = root.openNextFile(); + } + file.close(); +#endif +} + void fsInit() { #ifdef FSCom diff --git a/src/FSCommon.h b/src/FSCommon.h index be515b946..26d4f9f88 100644 --- a/src/FSCommon.h +++ b/src/FSCommon.h @@ -26,4 +26,6 @@ using namespace Adafruit_LittleFS_Namespace; #endif -void fsInit(); \ No newline at end of file +void fsInit(); +void listDir(const char * dirname, uint8_t levels); +void rmDir(const char * dirname); diff --git a/src/mesh/NodeDB.cpp b/src/mesh/NodeDB.cpp index c4ab8cad0..f70ffa1dd 100644 --- a/src/mesh/NodeDB.cpp +++ b/src/mesh/NodeDB.cpp @@ -91,7 +91,7 @@ bool NodeDB::resetRadioConfig() if (config.device.factory_reset) { DEBUG_MSG("Performing factory reset!\n"); // first, remove the "/prefs" (this removes most prefs) - FSCom.rmdir_r("/prefs"); + rmDir("/prefs"); // second, install default state (this will deal with the duplicate mac address issue) installDefaultDeviceState(); // third, write to disk From a1dc35023146625fc46d7d67d9747353da693289 Mon Sep 17 00:00:00 2001 From: Ben Meadors Date: Wed, 15 Jun 2022 11:44:37 -0500 Subject: [PATCH 16/20] Changed default baud to 115200 (#1517) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Thomas Göttgens --- bin/device-install.bat | 8 ++++---- bin/device-update.bat | 4 ++-- bin/device-update.sh | 4 ++-- bin/read-system-info.sh | 2 +- platformio.ini | 4 ++-- src/DebugConfiguration.h | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/bin/device-install.bat b/bin/device-install.bat index 88b24fc66..fe4a3870a 100644 --- a/bin/device-install.bat +++ b/bin/device-install.bat @@ -28,12 +28,12 @@ IF "__%FILENAME%__" == "____" ( ) IF EXIST %FILENAME% ( echo Trying to flash update %FILENAME%, but first erasing and writing system information" - %PYTHON% -m esptool --baud 921600 erase_flash - %PYTHON% -m esptool --baud 921600 write_flash 0x1000 system-info.bin + %PYTHON% -m esptool --baud 115200 erase_flash + %PYTHON% -m esptool --baud 115200 write_flash 0x1000 system-info.bin for %%f in (littlefs-*.bin) do ( - %PYTHON% -m esptool --baud 921600 write_flash 0x00390000 %%f + %PYTHON% -m esptool --baud 115200 write_flash 0x00390000 %%f ) - %PYTHON% -m esptool --baud 921600 write_flash 0x10000 %FILENAME% + %PYTHON% -m esptool --baud 115200 write_flash 0x10000 %FILENAME% ) else ( echo "Invalid file: %FILENAME%" goto HELP diff --git a/bin/device-update.bat b/bin/device-update.bat index 4cede3db9..7e8dc329c 100644 --- a/bin/device-update.bat +++ b/bin/device-update.bat @@ -28,9 +28,9 @@ IF "__%FILENAME%__" == "____" ( ) IF EXIST %FILENAME% ( echo Trying to flash update %FILENAME% - %PYTHON% -m esptool --baud 921600 write_flash 0x10000 %FILENAME% + %PYTHON% -m esptool --baud 115200 write_flash 0x10000 %FILENAME% echo Erasing the otadata partition, which will turn off flash flippy-flop and force the first image to be used - %PYTHON% -m esptool --baud 921600 erase_region 0xe000 0x2000 + %PYTHON% -m esptool --baud 115200 erase_region 0xe000 0x2000 ) else ( echo "Invalid file: %FILENAME%" goto HELP diff --git a/bin/device-update.sh b/bin/device-update.sh index dda05f005..b4e054196 100755 --- a/bin/device-update.sh +++ b/bin/device-update.sh @@ -44,9 +44,9 @@ shift "$((OPTIND-1))" if [ -f "${FILENAME}" ]; then echo "Trying to flash update ${FILENAME}." - $PYTHON -m esptool --baud 921600 write_flash 0x10000 ${FILENAME} + $PYTHON -m esptool --baud 115200 write_flash 0x10000 ${FILENAME} echo "Erasing the otadata partition, which will turn off flash flippy-flop and force the first image to be used" - $PYTHON -m esptool --baud 921600 erase_region 0xe000 0x2000 + $PYTHON -m esptool --baud 115200 erase_region 0xe000 0x2000 else echo "Invalid file: ${FILENAME}" show_help diff --git a/bin/read-system-info.sh b/bin/read-system-info.sh index f116087b2..b9e9c8072 100755 --- a/bin/read-system-info.sh +++ b/bin/read-system-info.sh @@ -1,3 +1,3 @@ #!/usr/bin/env bash -esptool.py --baud 921600 read_flash 0x1000 0xf000 system-info.img +esptool.py --baud 115200 read_flash 0x1000 0xf000 system-info.img diff --git a/platformio.ini b/platformio.ini index 60e9fe68e..5a69e24ab 100644 --- a/platformio.ini +++ b/platformio.ini @@ -38,7 +38,7 @@ build_flags = -Wno-missing-field-initializers -DTINYGPS_OPTION_NO_CUSTOM_FIELDS -DPB_ENABLE_MALLOC=1 -monitor_speed = 921600 +monitor_speed = 115200 lib_deps = https://github.com/meshtastic/esp8266-oled-ssd1306.git#53580644255b48ebb7a737343c6b4e71c7e11cf2 ; ESP8266_SSD1306 @@ -93,7 +93,7 @@ extends = arduino_base platform = espressif32@3.5.0 build_src_filter = ${arduino_base.build_src_filter} - -upload_speed = 921600 +upload_speed = 115200 debug_init_break = tbreak setup # Remove -DMYNEWT_VAL_BLE_HS_LOG_LVL=LOG_LEVEL_CRITICAL for low level BLE logging. diff --git a/src/DebugConfiguration.h b/src/DebugConfiguration.h index 4397dfe0b..53ea303a2 100644 --- a/src/DebugConfiguration.h +++ b/src/DebugConfiguration.h @@ -10,7 +10,7 @@ #ifdef CONSOLE_MAX_BAUD #define SERIAL_BAUD CONSOLE_MAX_BAUD #else -#define SERIAL_BAUD 921600 // Serial debug baud rate +#define SERIAL_BAUD 115200 // Serial debug baud rate #endif #include "SerialConsole.h" From 0943e5f500c3bbc6ebd9bb6465836cee68f4290b Mon Sep 17 00:00:00 2001 From: caveman99 Date: Wed, 15 Jun 2022 17:40:08 +0000 Subject: [PATCH 17/20] [create-pull-request] automated change --- protobufs | 2 +- src/mesh/generated/mesh.pb.h | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/protobufs b/protobufs index 9b0148e86..a7bbc3586 160000 --- a/protobufs +++ b/protobufs @@ -1 +1 @@ -Subproject commit 9b0148e86b28d995c56e3aca209f5e2af6360c5b +Subproject commit a7bbc358664c83f5148bae2a502cb294e7cafee5 diff --git a/src/mesh/generated/mesh.pb.h b/src/mesh/generated/mesh.pb.h index 893760abd..bbc79896e 100644 --- a/src/mesh/generated/mesh.pb.h +++ b/src/mesh/generated/mesh.pb.h @@ -4,6 +4,7 @@ #ifndef PB_MESH_PB_H_INCLUDED #define PB_MESH_PB_H_INCLUDED #include +#include "localonly.pb.h" #include "portnums.pb.h" #include "telemetry.pb.h" @@ -630,6 +631,7 @@ typedef struct _FromRadio { union { MyNodeInfo my_info; NodeInfo node_info; + LocalConfig config; LogRecord log_record; uint32_t config_complete_id; bool rebooted; @@ -819,6 +821,7 @@ extern "C" { #define FromRadio_id_tag 1 #define FromRadio_my_info_tag 3 #define FromRadio_node_info_tag 4 +#define FromRadio_config_tag 6 #define FromRadio_log_record_tag 7 #define FromRadio_config_complete_id_tag 8 #define FromRadio_rebooted_tag 9 @@ -968,6 +971,7 @@ X(a, STATIC, SINGULAR, UENUM, level, 4) X(a, STATIC, SINGULAR, UINT32, id, 1) \ X(a, STATIC, ONEOF, MESSAGE, (payloadVariant,my_info,my_info), 3) \ X(a, STATIC, ONEOF, MESSAGE, (payloadVariant,node_info,node_info), 4) \ +X(a, STATIC, ONEOF, MESSAGE, (payloadVariant,config,config), 6) \ X(a, STATIC, ONEOF, MESSAGE, (payloadVariant,log_record,log_record), 7) \ X(a, STATIC, ONEOF, UINT32, (payloadVariant,config_complete_id,config_complete_id), 8) \ X(a, STATIC, ONEOF, BOOL, (payloadVariant,rebooted,rebooted), 9) \ @@ -976,6 +980,7 @@ X(a, STATIC, ONEOF, MESSAGE, (payloadVariant,packet,packet), 11) #define FromRadio_DEFAULT NULL #define FromRadio_payloadVariant_my_info_MSGTYPE MyNodeInfo #define FromRadio_payloadVariant_node_info_MSGTYPE NodeInfo +#define FromRadio_payloadVariant_config_MSGTYPE LocalConfig #define FromRadio_payloadVariant_log_record_MSGTYPE LogRecord #define FromRadio_payloadVariant_packet_MSGTYPE MeshPacket From 8684fd1c49957d20f81ddc104eaf5ed1492be57e Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Wed, 15 Jun 2022 19:00:40 -0700 Subject: [PATCH 18/20] Bump to .18 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index dd4911456..533ab8800 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 17 +build = 18 From b40abbf3ad06b8f93961deb8a50e4b12d6bd5c47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thomas=20G=C3=B6ttgens?= Date: Thu, 16 Jun 2022 12:22:01 +0200 Subject: [PATCH 19/20] Tryfix Portduino Radiolib --- src/mesh/RadioLibInterface.cpp | 13 +++++++++++++ src/mesh/RadioLibInterface.h | 6 +++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/mesh/RadioLibInterface.cpp b/src/mesh/RadioLibInterface.cpp index afec06ba0..a7cadf2ad 100644 --- a/src/mesh/RadioLibInterface.cpp +++ b/src/mesh/RadioLibInterface.cpp @@ -11,6 +11,17 @@ // FIXME, we default to 4MHz SPI, SPI mode 0, check if the datasheet says it can really do that static SPISettings spiSettings(4000000, MSBFIRST, SPI_MODE0); +#ifdef PORTDUINO + +void LockingModule::SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes) +{ + concurrency::LockGuard g(spiLock); + + Module::SPItransfer(cmd, reg, dataOut, dataIn, numBytes); +} + +#else + void LockingModule::SPIbeginTransaction() { spiLock->lock(); @@ -25,6 +36,8 @@ void LockingModule::SPIendTransaction() Module::SPIendTransaction(); } +#endif + RadioLibInterface::RadioLibInterface(RADIOLIB_PIN_TYPE cs, RADIOLIB_PIN_TYPE irq, RADIOLIB_PIN_TYPE rst, RADIOLIB_PIN_TYPE busy, SPIClass &spi, PhysicalLayer *_iface) : NotifiedWorkerThread("RadioIf"), module(cs, irq, rst, busy, spi, spiSettings), iface(_iface) diff --git a/src/mesh/RadioLibInterface.h b/src/mesh/RadioLibInterface.h index f940f8908..08aa84811 100644 --- a/src/mesh/RadioLibInterface.h +++ b/src/mesh/RadioLibInterface.h @@ -40,9 +40,13 @@ class LockingModule : public Module : Module(cs, irq, rst, gpio, spi, spiSettings) { } - + +#ifdef PORTDUINO + void SPItransfer(uint8_t cmd, uint8_t reg, uint8_t *dataOut, uint8_t *dataIn, uint8_t numBytes) override; +#else void SPIbeginTransaction() override; void SPIendTransaction() override; +#endif }; class RadioLibInterface : public RadioInterface, protected concurrency::NotifiedWorkerThread From 3c6a2f7bb682dfe547344e1fe3d0e90a0e227f3b Mon Sep 17 00:00:00 2001 From: Jm Casler Date: Thu, 16 Jun 2022 05:24:08 -0700 Subject: [PATCH 20/20] Bump to 1.3.19 --- version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.properties b/version.properties index 533ab8800..6aa1edff3 100644 --- a/version.properties +++ b/version.properties @@ -1,4 +1,4 @@ [VERSION] major = 1 minor = 3 -build = 18 +build = 19