tools/mpremote: Add hashing ability and use for recursive copy.

Changes in this commit:
- Adds transport API `fs_hashfile` to compute the hash of a file with given
  algorithm.
- Adds commands `mpremote <...>sum file` to compute and print hashes of
  various algorithms.
- Adds shortcut `mpremote sha256sum file`.
- Uses the hash computation to improve speed of recursive file copy to
  avoid copying a file where the target is identical.

For recursive copy, if possible it will use the board's support (e.g.
built-in hashlib or hashlib from micropython-lib), but will fall back to
downloading the file and using the local implementation.

This work was funded through GitHub Sponsors.

Signed-off-by: Jim Mussared <jim.mussared@gmail.com>
Signed-off-by: Damien George <damien@micropython.org>

tools/mpremote/README.md

@@ -20,7 +20,7 @@ The full list of supported commands are:
     mpremote exec <string>            -- execute the string
     mpremote run <file>               -- run the given local script
     mpremote fs <command> <args...>   -- execute filesystem commands on the device
-                                         command may be: cat, ls, cp, rm, mkdir, rmdir
+                                         command may be: cat, ls, cp, rm, mkdir, rmdir, sha256sum
                                          use ":" as a prefix to specify a file on the device
     mpremote repl                     -- enter REPL
                                          options:
@@ -78,6 +78,7 @@ Examples:
     mpremote cp :main.py .
     mpremote cp main.py :
     mpremote cp -r dir/ :
+    mpremote sha256sum :main.py
     mpremote mip install aioble
     mpremote mip install github:org/repo@branch
     mpremote mip install gitlab:org/repo@branch

tools/mpremote/mpremote/commands.py

@@ -1,3 +1,4 @@
+import hashlib
 import os
 import sys
 import tempfile
@@ -127,7 +128,7 @@ def _remote_path_basename(a):
     return a.rsplit("/", 1)[-1]
 
 
-def do_filesystem_cp(state, src, dest, multiple):
+def do_filesystem_cp(state, src, dest, multiple, check_hash=False):
     if dest.startswith(":"):
         dest_exists = state.transport.fs_exists(dest[1:])
         dest_isdir = dest_exists and state.transport.fs_isdir(dest[1:])
@@ -159,6 +160,19 @@ def do_filesystem_cp(state, src, dest, multiple):
         if dest_isdir:
             dest = ":" + _remote_path_join(dest[1:], filename)
 
+        # Skip copy if the destination file is identical.
+        if check_hash:
+            try:
+                remote_hash = state.transport.fs_hashfile(dest[1:], "sha256")
+                source_hash = hashlib.sha256(data).digest()
+                # remote_hash will be None if the device doesn't support
+                # hashlib.sha256 (and therefore won't match).
+                if remote_hash == source_hash:
+                    print("Up to date:", dest[1:])
+                    return
+            except OSError:
+                pass
+
         # Write to remote.
         state.transport.fs_writefile(dest[1:], data, progress_callback=show_progress_bar)
     else:
@@ -274,7 +288,7 @@ def do_filesystem_recursive_cp(state, src, dest, multiple):
         else:
             dest_path_joined = os.path.join(dest, *dest_path_split)
 
-        do_filesystem_cp(state, src_path_joined, dest_path_joined, multiple=False)
+        do_filesystem_cp(state, src_path_joined, dest_path_joined, multiple=False, check_hash=True)
 
 
 def do_filesystem(state, args):
@@ -333,6 +347,9 @@ def do_filesystem(state, args):
             state.transport.fs_rmdir(path)
         elif command == "touch":
             state.transport.fs_touchfile(path)
+        elif command.endswith("sum") and command[-4].isdigit():
+            digest = state.transport.fs_hashfile(path, command[:-3])
+            print(digest.hex())
         elif command == "cp":
             if args.recursive:
                 do_filesystem_recursive_cp(state, path, cp_dest, len(paths) > 1)

tools/mpremote/mpremote/main.py

@@ -190,7 +190,9 @@ def argparse_filesystem():
         "enable verbose output (defaults to True for all commands except cat)",
     )
     cmd_parser.add_argument(
-        "command", nargs=1, help="filesystem command (e.g. cat, cp, ls, rm, rmdir, touch)"
+        "command",
+        nargs=1,
+        help="filesystem command (e.g. cat, cp, sha256sum, ls, rm, rmdir, touch)",
     )
     cmd_parser.add_argument("path", nargs="+", help="local and remote paths")
     return cmd_parser
@@ -308,12 +310,13 @@ _BUILTIN_COMMAND_EXPANSIONS = {
     },
     # Filesystem shortcuts (use `cp` instead of `fs cp`).
     "cat": "fs cat",
-    "ls": "fs ls",
     "cp": "fs cp",
-    "rm": "fs rm",
+    "ls": "fs ls",
-    "touch": "fs touch",
     "mkdir": "fs mkdir",
+    "rm": "fs rm",
     "rmdir": "fs rmdir",
+    "sha256sum": "fs sha256sum",
+    "touch": "fs touch",
     # Disk used/free.
     "df": [
         "exec",

tools/mpremote/mpremote/transport.py

@@ -24,7 +24,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
-import ast, os, sys
+import ast, hashlib, os, sys
 from collections import namedtuple
 
 
@@ -174,3 +174,20 @@ class Transport:
             self.exec("f=open('%s','a')\nf.close()" % path)
         except TransportError as e:
             raise _convert_filesystem_error(e, path) from None
+
+    def fs_hashfile(self, path, algo, chunk_size=256):
+        try:
+            self.exec("import hashlib\nh = hashlib.{algo}()".format(algo=algo))
+        except TransportError:
+            # hashlib (or hashlib.{algo}) not available on device. Do the hash locally.
+            data = self.fs_readfile(path, chunk_size=chunk_size)
+            return getattr(hashlib, algo)(data).digest()
+        try:
+            self.exec(
+                "buf = memoryview(bytearray({chunk_size}))\nwith open('{path}', 'rb') as f:\n while True:\n  n = f.readinto(buf)\n  if n == 0:\n   break\n  h.update(buf if n == {chunk_size} else buf[:n])\n".format(
+                    chunk_size=chunk_size, path=path
+                )
+            )
+            return self.eval("h.digest()")
+        except TransportExecError as e:
+            raise _convert_filesystem_error(e, path) from None