Introduced support for resumable upload in OneDrive API, chunk upload in SharePoint API

vgrem · vgrem · commit 68e7e2249e75 · 2020-02-03T13:06:36.000+02:00
diff --git a/.travis.yml b/.travis.yml
@@ -1,7 +1,6 @@
 language: python
 python:
   - "2.7"
-  - "3.3"
   - "3.4"
   - "3.5"
   - "3.6"
diff --git a/README.md b/README.md
@@ -26,7 +26,7 @@ pip install Office365-REST-Python-Client
 # Working with SharePoint API
 
 The list of supported API versions: 
--   [SharePoint 2013 REST API or above](https://msdn.microsoft.com/en-us/library/office/jj860569.aspx) and above 
+-   [SharePoint 2013 REST API](https://msdn.microsoft.com/en-us/library/office/jj860569.aspx) and above 
 -   SharePoint Online & OneDrive for Business REST API
 
 #### Authentication
diff --git a/examples/onedrive/export_files.py b/examples/onedrive/export_files.py
@@ -44,6 +44,6 @@ def get_file_content(name):
 for file_name in get_file_names():
     print("Reading local file: {0}".format(file_name))
     file_content = get_file_content(file_name)
-    uploaded_file = target_drive.root.upload(file_name, file_content)
+    uploaded_file = target_drive.root.execute(file_name, file_content)
     client.execute_query()
     print("File has been uploaded into: {0}".format(uploaded_file.webUrl))
diff --git a/examples/sharepoint/file_upload_session.py b/examples/sharepoint/file_upload_session.py
@@ -1,51 +1,10 @@
-import os
-import uuid
-
-from office365.sharepoint.file_creation_information import FileCreationInformation
 from settings import settings
 from office365.runtime.auth.authentication_context import AuthenticationContext
 from office365.sharepoint.client_context import ClientContext
 
 
-def read_in_chunks(file_object, size=1024):
-    """Lazy function (generator) to read a file piece by piece.
-    Default chunk size: 1k."""
-    while True:
-        data = file_object.read(size)
-        if not data:
-            break
-        yield data
-
-
-def upload_file_session(context, local_path, target_folder_url,chunk_size):
-    upload_id = str(uuid.uuid4())
-    f = open(local_path, 'rb')
-    st = os.stat(local_path)
-
-    # 1. create an empty file first
-    info = FileCreationInformation()
-    info.content = ""
-    info.url = os.path.basename(local_path)
-    info.overwrite = True
-    target_folder = context.web.get_folder_by_server_relative_url(target_folder_url)
-    target_file = target_folder.files.add(info)
-    context.execute_query()
-
-    # 2. upload a file via session
-    target_file_url = os.path.basename(local_path)
-    f_pos = 0
-    for piece in read_in_chunks(f, size=chunk_size):
-        if f_pos == 0:
-            upload_result = target_folder.files.get_by_url(target_file_url).start_upload(upload_id, piece)
-            context.execute_query()
-        elif f_pos + len(piece) < st.st_size:
-            upload_result = target_folder.files.get_by_url(target_file_url).continue_upload(upload_id, f_pos,
-                                                                                            piece)
-            context.execute_query()
-        else:
-            upload_result = target_folder.files.get_by_url(target_file_url).finish_upload(upload_id, f_pos, piece)
-            context.execute_query()
-        f_pos += len(piece)
+def print_upload_progress(offset):
+    print ("Uploaded '{0}' bytes...".format(offset))
 
 
 if __name__ == '__main__':
@@ -55,7 +14,13 @@ def upload_file_session(context, local_path, target_folder_url,chunk_size):
                                        password=settings['user_credentials']['password']):
         ctx = ClientContext(site_url, ctx_auth)
 
-        size_4k = 1024 * 4
-        path = "../data/SharePoint User Guide.docx"
+        # size_4k = 1024 * 4
+        size_1Mb = 1000000
+        local_path = "../../tests/data/big_buck_bunny.mp4"
         target_url = "/Shared Documents"
-        upload_file_session(ctx, path, target_url, size_4k)
+        # result_file = upload_file_session(ctx, target_url, local_path, size_1Mb)
+
+        result_file = ctx.web.get_folder_by_server_relative_url(target_url)\
+            .files.create_upload_session(local_path, size_1Mb, print_upload_progress)
+        ctx.execute_query()
+        print ('File {0} has been uploaded successfully'.format(result_file.properties['ServerRelativeUrl']))
diff --git a/office365/onedrive/columnDefinition.py b/office365/onedrive/columnDefinition.py
@@ -0,0 +1,5 @@
+from office365.onedrive.baseItem import BaseItem
+
+
+class ColumnDefinition(BaseItem):
+    """"""
diff --git a/office365/onedrive/file_upload.py b/office365/onedrive/file_upload.py
@@ -0,0 +1,57 @@
+import os
+
+from office365.onedrive.driveItemUploadableProperties import DriveItemUploadableProperties
+from office365.runtime.utilities.http_method import HttpMethod
+from office365.runtime.utilities.request_options import RequestOptions
+
+
+def read_in_chunks(file_object, chunk_size=1024):
+    """Lazy function (generator) to read a file piece by piece.
+    Default chunk size: 1k."""
+    while True:
+        data = file_object.read(chunk_size)
+        if not data:
+            break
+        yield data
+
+
+class ResumableFileUpload(object):
+    """Create an upload session to allow your app to upload files up to the maximum file size. An upload session
+    allows your app to upload ranges of the file in sequential API requests, which allows the transfer to be resumed
+    if a connection is dropped while the upload is in progress. """
+    def __init__(self, target_folder, source_path, chunk_size):
+        self._target_folder = target_folder
+        self._chunk_size = chunk_size
+        self._source_path = source_path
+        self._uploaded_bytes = 0
+
+    def execute(self, chunk_uploaded=None):
+        ctx = self._target_folder.context
+        file_name = os.path.basename(self._source_path)
+        # 1. create an empty file
+        target_item = self._target_folder.upload(file_name, None)
+        ctx.execute_query()
+
+        # 2. create upload session
+        item = DriveItemUploadableProperties()
+        item.name = file_name
+        session_result = target_item.create_upload_session(item)
+        ctx.execute_query()
+
+        # 3. start upload
+        fh = open(self._source_path, 'rb')
+        st = os.stat(self._source_path)
+        f_pos = 0
+        for piece in read_in_chunks(fh, chunk_size=self._chunk_size):
+            req = RequestOptions(session_result.value.uploadUrl)
+            req.method = HttpMethod.Put
+            req.set_header('Content-Length', str(len(piece)))
+            req.set_header('Content-Range', 'bytes {0}-{1}/{2}'.format(f_pos, (f_pos + len(piece) - 1), st.st_size))
+            req.set_header('Accept', '*/*')
+            req.data = piece
+            resp = ctx.execute_request_direct(req)
+            f_pos += len(piece)
+            if chunk_uploaded is not None:
+                chunk_uploaded(f_pos)
+
+        return target_item
diff --git a/office365/runtime/client_request.py b/office365/runtime/client_request.py
@@ -16,10 +16,12 @@ def __init__(self, context):
         self.context = context
         self.__queries = []
         self.__resultObjects = {}
+        self.__events_list = {}
 
     def clear(self):
         self.__queries = []
         self.__resultObjects = {}
+        self.__events_list = {}
 
     def build_request(self, query):
         request = RequestOptions(query.url)
@@ -50,9 +52,13 @@ def execute_query(self):
         try:
             for query in self.__queries:
                 request = self.build_request(query)
+                if 'BeforeExecuteQuery' in self.__events_list:
+                    self.__events_list['BeforeExecuteQuery'](request)
                 response = self.execute_request_direct(request)
                 result_object = self.__resultObjects.get(query)
                 self.process_response(response, result_object)
+                if 'AfterExecuteQuery' in self.__events_list:
+                    self.__events_list['AfterExecuteQuery'](result_object)
         finally:
             self.clear()
 
@@ -122,6 +128,12 @@ def add_query(self, query, result_object=None):
         if result_object is not None:
             self.__resultObjects[query] = result_object
 
+    def before_execute_query(self, event):
+        self.__events_list['BeforeExecuteQuery'] = event
+
+    def after_execute_query(self, event):
+        self.__events_list['AfterExecuteQuery'] = event
+
     def validate_response(self, response):
         try:
             response.raise_for_status()
diff --git a/office365/sharepoint/client_context.py b/office365/sharepoint/client_context.py
@@ -4,7 +4,6 @@
 from office365.runtime.context_web_information import ContextWebInformation
 from office365.runtime.odata.json_light_format import JsonLightFormat
 from office365.runtime.odata.odata_metadata_level import ODataMetadataLevel
-from office365.runtime.utilities.http_method import HttpMethod
 from office365.runtime.utilities.request_options import RequestOptions
 from office365.sharepoint.site import Site
 from office365.sharepoint.web import Web
diff --git a/office365/sharepoint/file.py b/office365/sharepoint/file.py
@@ -196,11 +196,7 @@ def finish_upload(self, upload_id, file_offset, content):
 
     @staticmethod
     def save_binary(ctx, server_relative_url, content):
-        try:
-            from urllib import quote  # Python 2.X
-        except ImportError:
-            from urllib.parse import quote  # Python 3+
-        server_relative_url = quote(server_relative_url)
+        """Uploads a file"""
         url = r"{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(
             ctx.serviceRootUrl, server_relative_url)
         request = RequestOptions(url)
diff --git a/office365/sharepoint/file_collection.py b/office365/sharepoint/file_collection.py
@@ -3,6 +3,7 @@
 from office365.runtime.resource_path_service_operation import ResourcePathServiceOperation
 from office365.runtime.utilities.http_method import HttpMethod
 from office365.sharepoint.file import File
+from office365.sharepoint.upload_session import UploadSession
 
 
 class FileCollection(ClientObjectCollection):
@@ -11,6 +12,12 @@ class FileCollection(ClientObjectCollection):
     def __init__(self, context, resource_path=None):
         super(FileCollection, self).__init__(context, File, resource_path)
 
+    def create_upload_session(self, source_path, chunk_size, chunk_uploaded=None):
+        """Upload a file as multiple chunks"""
+        session = UploadSession(source_path, chunk_size, chunk_uploaded)
+        session.build_query(self)
+        return session.file
+
     def add(self, file_creation_information):
         """Creates a File resource"""
         file_new = File(self.context)
diff --git a/office365/sharepoint/upload_session.py b/office365/sharepoint/upload_session.py
@@ -0,0 +1,65 @@
+import os
+import uuid
+
+from office365.runtime.client_result import ClientResult
+from office365.sharepoint.file import File
+from office365.sharepoint.file_creation_information import FileCreationInformation
+
+
+def read_in_chunks(file_object, size=1024):
+    """Lazy function (generator) to read a file piece by piece.
+    Default chunk size: 1k."""
+    while True:
+        data = file_object.read(size)
+        if not data:
+            break
+        yield data
+
+
+class UploadSession(object):
+    def __init__(self, source_path, chunk_size, chunk_uploaded):
+        self._chunk_size = chunk_size
+        self._upload_id = str(uuid.uuid4())
+        self._source_path = source_path
+        self._chunk_uploaded = chunk_uploaded
+        self._target_file = None
+        self._uploaded_bytes = 0
+
+    def build_query(self, files):
+        st = os.stat(self._source_path)
+        file_name = os.path.basename(self._source_path)
+
+        # 1. create an empty target file
+        info = FileCreationInformation()
+        info.url = file_name
+        info.overwrite = True
+        self._target_file = files.add(info)
+
+        # 2. upload a file in chunks
+        f_pos = 0
+        fh = open(self._source_path, 'rb')
+        for piece in read_in_chunks(fh, size=self._chunk_size):
+            if f_pos == 0:
+                upload_result = files.get_by_url(file_name).start_upload(self._upload_id, piece)
+            elif f_pos + len(piece) < st.st_size:
+                upload_result = files.get_by_url(file_name).continue_upload(self._upload_id, f_pos, piece)
+            else:
+                self._target_file = files.get_by_url(file_name).finish_upload(self._upload_id, f_pos, piece)
+            f_pos += len(piece)
+
+        if self._chunk_uploaded is not None:
+            files.context.pending_request.after_execute_query(self._process_chunk_upload)
+
+    def _process_chunk_upload(self, result_object):
+        if isinstance(result_object, ClientResult):
+            if 'StartUpload' in result_object.value:
+                self._uploaded_bytes = int(result_object.value['StartUpload'])
+            elif 'ContinueUpload' in result_object.value:
+                self._uploaded_bytes = int(result_object.value['ContinueUpload'])
+        elif isinstance(result_object, File):
+            self._uploaded_bytes = int(result_object.properties['Length'])
+        self._chunk_uploaded(self._uploaded_bytes)
+
+    @property
+    def file(self):
+        return self._target_file
diff --git a/tests/test_onedrive_driveItem.py b/tests/test_onedrive_driveItem.py
@@ -1,9 +1,7 @@
 import os
 import uuid
 from unittest import TestCase
-from office365.onedrive.driveItemUploadableProperties import DriveItemUploadableProperties
-from office365.runtime.utilities.http_method import HttpMethod
-from office365.runtime.utilities.request_options import RequestOptions
+from office365.onedrive.file_upload import ResumableFileUpload
 from settings import settings
 
 from office365.graphClient import GraphClient
@@ -18,16 +16,6 @@ def get_token(auth_ctx):
     return token
 
 
-def read_in_chunks(file_object, chunk_size=1024):
-    """Lazy function (generator) to read a file piece by piece.
-    Default chunk size: 1k."""
-    while True:
-        data = file_object.read(chunk_size)
-        if not data:
-            break
-        yield data
-
-
 def create_listDrive(client):
     list_info = {
         "displayName": "Lib_" + uuid.uuid4().hex,
@@ -70,6 +58,13 @@ def test2_upload_file(self):
         self.client.execute_query()
         self.assertIsNotNone(self.target_file.webUrl)
 
+    def test21_upload_file_session(self):
+        file_name = "big_buck_bunny.mp4"
+        local_path = "{0}/data/{1}".format(os.path.dirname(__file__), file_name)
+        uploader = ResumableFileUpload(self.target_drive.root, local_path, 1000000)
+        uploader.execute()
+        print("{0} bytes has been uploaded".format(0))
+
     def test3_download_file(self):
         result = self.__class__.target_file.download()
         self.client.execute_query()
@@ -97,31 +92,3 @@ def test6_delete_file(self):
         self.client.execute_query()
 
         self.assertEqual(before_count - 1, len(items))
-
-    def test7_upload_file_session(self):
-        file_name = "big_buck_bunny.mp4"
-        path = "{0}/data/{1}".format(os.path.dirname(__file__), file_name)
-        # 1. create a file
-        target_item = self.target_drive.root.upload(file_name, None)
-        self.client.execute_query()
-        self.assertIsNotNone(target_item.properties['id'])
-        # 2. create upload session
-        item = DriveItemUploadableProperties()
-        item.name = file_name
-        session_result = target_item.create_upload_session(item)
-        self.client.execute_query()
-        self.assertIsNotNone(session_result.value)
-        # 3. start upload
-        f = open(path, 'rb')
-        st = os.stat(path)
-        f_pos = 0
-        for piece in read_in_chunks(f, chunk_size=1000000):
-            req = RequestOptions(session_result.value.uploadUrl)
-            req.method = HttpMethod.Put
-            req.set_header('Content-Length', str(len(piece)))
-            req.set_header('Content-Range', 'bytes {0}-{1}/{2}'.format(f_pos, (f_pos + len(piece) - 1), st.st_size))
-            req.set_header('Accept', '*/*')
-            req.data = piece
-            resp = self.client.execute_request_direct(req)
-            self.assertTrue(resp.ok)
-            f_pos += len(piece)

-Original file line number
+Diff line change
 language: python
 python:
   - "2.7"
 -  - "3.3"
   - "3.4"
   - "3.5"
   - "3.6"