large-file-upload: add option to set binary buffer size (#32)

* large-file-upload: add option to set binary buffer size

The command line argument -B/--binary-size can now be used to set the
binary buffer size that will be used when transmitting binary data to
Notehub. This gives more control to the user and is beneficial in case
there are any application/firmware issues with sending large binary data

* large-file-upload: update code for -B option based on feedback from PR

The code has been updated so that if the _binaryBuffSize (which is specified
via the -B option) is larger than the notecard's binary max, the max value
is used

python-large-file-upload/main.py

@@ -41,7 +41,7 @@ def parseCommandLineArgs():
     p.add("-i", "--include-file-name", help="Add file name as query parameter to web request", action='store_true')
     p.add("-e", "--measure-elapsed-time", help="Measure how long the file transfer process takes", action='store_true')
     p.add("--legacy", help="Use legacy method to upload file. Uses base64 encoding in web transaction payloads", action='store_true')
-
+    p.add("-B", "--binary-size", help="Size of binary data to send in each transaction", type=int)
 
     opts = p.parse_args()
     hub_config = {}
@@ -124,6 +124,9 @@ def main():
         fileName = os.path.basename(os.path.normpath(opts.file))
         uploader.setFileName(fileName)
 
+    if opts.binary_size:
+        uploader.setBinaryBuffSize(opts.binary_size)
+
     startTime = 0
     fileSizeInBytes = 0
     with open(opts.file, 'rb') as f:

python-large-file-upload/notecardDataTransfer.py

@@ -27,7 +27,8 @@ def __init__(self, card, route, timeout=DEFAULT_WEB_TRANSACTION_TIMEOUT_SEC, pri
         self.webReqRoot['route'] = route
         self.webReqRoot['seconds'] = timeout
         self._fileName = None
-
+        self._binaryBuffSize = None
+
     def setFileName(self, fileName):
         self._fileName = fileName
 
@@ -95,13 +96,21 @@ def _writeAndFlushBytes(self, data: io.IOBase):
         while bytesSent < totalBytes:
             binary_helpers.binary_store_reset(self._card)
             rsp = self._sendRequest("card.binary")
-            max = rsp.get("max", 0)
-
-            buffer = bytearray(max)
-            numBytes = data.readinto(buffer)
 
+            # First set the buffer size to be equal to the max binary buffer size supported by the notecard,
+            # then check if the user has specified a smaller buffer size through the -B/--binary-size flag
+            buffSize = rsp.get("max", 0)
+            if self._binaryBuffSize is not None:
+                buffSize = min(self._binaryBuffSize, buffSize)
+
+            # Create a buffer of the specified size
+            buffer = bytearray(buffSize)
+
+            # Read the data from the file and store it in the notecard's binary store
+            numBytes = data.readinto(buffer)
             binary_helpers.binary_store_transmit(self._card, buffer[0:numBytes], 0)
-
+
+            # Send the binary data to notehub
             self._writeWebReqBinary(bytesSent, totalBytes)
 
             bytesSent += numBytes
@@ -127,6 +136,9 @@ def _unsetTempContinuousMode(self):
         req = {"req":"hub.set", "off":True}
         self._sendRequest(req)
 
+    def setBinaryBuffSize(self, binaryBuffSize):
+        self._binaryBuffSize = binaryBuffSize
+
 
 import binascii
 class BinaryDataUploaderLegacy(BinaryDataUploader):