Magic uploads in the Ubuntu One files REST API

Finally I’ve had a chance to document “magic uploads” in the Ubuntu One REST API. The documentation isn’t published yet, but it’s a neat little feature — when uploading a file to U1, you can pass a couple of hashes calculated from the file content instead of the actual file content, and if that file’s already in U1, it’ll “upload” without you having to send all the content on the wire, which is a lot quicker. I can’t do better in describing it than jdo did when he first wrote about it (a year ago. I know, I know, sorry.) We’ve had this for ages, and the U1 apps on mobile and so on use it, but I haven’t had a chance to document it so everyone can use it. Docs will be at https://one.ubuntu.com/developer/files/store_files/cloud fairly soon, but I thought it might be useful to add a little test Python script I wrote to confirm that I understood the thing properly.

import os, json
try:
    from ubuntuone.platform.credentials import CredentialsManagementTool
except ImportError:
    CredentialsManagementTool = None
import urlparse, urllib
from hashlib import sha1
from ubuntuone.couch import auth

CONTENT = "This is the content of the file.\n" * 1000

_u1creds = None
def get_ubuntuone_credentials_synchronous():
    "find_credentials is async, but be sync here with a temporary mainloop"
    from gi.repository import GObject
    from dbus.mainloop.glib import DBusGMainLoop
    from ubuntuone.platform.credentials import CredentialsManagementTool

    global _u1creds

    DBusGMainLoop(set_as_default=True)
    loop = GObject.MainLoop()

    def quit(result):
        global _u1creds
        loop.quit()
        if result:
            _u1creds = result

    cd = CredentialsManagementTool()
    d = cd.find_credentials()
    d.addCallbacks(quit)
    loop.run()
    return _u1creds

def make_rest_file(path, content, creds):
    url = urlparse.urljoin("https://one.ubuntu.com/api/file_storage/v1/",
        urllib.quote(path, safe="~/"))
    # First, create file
    body = {"kind": "file"}
    result_header, result_body = auth.request(url=url, sigmeth="HMAC_SHA1",
        http_method="PUT", request_body=json.dumps(body), 
        access_token = creds["token"], token_secret=creds["token_secret"], 
        consumer_key=creds["consumer_key"],
        consumer_secret=creds["consumer_secret"])
    result_body = json.loads(result_body)
    # now, PUT the body of the file
    content_root = "https://files.one.ubuntu.com/"
    put_url = urlparse.urljoin(content_root, urllib.quote(
        result_body["content_path"], safe="~/"))
    put_result_header, put_result_body = auth.request(url=put_url, 
        sigmeth="PLAINTEXT", http_method="PUT", request_body=CONTENT, 
        access_token = creds["token"], token_secret=creds["token_secret"], 
        consumer_key=creds["consumer_key"],
        consumer_secret=creds["consumer_secret"])
    return put_result_header["status"] in ("200", "201")

def make_rest_file_by_magic(path, content, creds):
    url = urlparse.urljoin("https://one.ubuntu.com/api/file_storage/v1/",
        urllib.quote(path, safe="~/"))
    # Note that we do not actually include the CONTENT of the file 
    # here in the PUT.
    # We just upload by passing the hash and magic_hash.
    file_hash = sha1()
    magic_hash = sha1("Ubuntu One")
    file_hash.update(content)
    magic_hash.update(content)
    body = {
        "kind": "file",
        "hash": "sha1:%s" % file_hash.hexdigest(),
        "magic_hash": "magic_hash:%s" % magic_hash.hexdigest()
    }

    result_header, result_body = auth.request(url=url, sigmeth="HMAC_SHA1",
        http_method="PUT", request_body=json.dumps(body), 
        access_token = creds["token"],
        token_secret=creds["token_secret"], 
        consumer_key=creds["consumer_key"],
        consumer_secret=creds["consumer_secret"])
    result_body = json.loads(result_body)
    # if failed, will be a 400 Bad Request with body 
    # {u'error': u'The content could not be reused.'}
    return result_header["status"] in ("200", "201")


def main():
    credentials = get_ubuntuone_credentials_synchronous()
    # real strings, not dbus
    credentials = dict([(str(k),str(v)) for k,v in credentials.items()])

    print "Creating a file with the REST API"
    success = make_rest_file("~/Ubuntu One/u1-test-magic-uploads.txt", 
        CONTENT, credentials)
    if not success:
        print "Failed, somehow"
        return
    print "Succeeded."
    print ("Now, try uploading a new file, but by magic, "
        "so we will not actually upload it.")
    success = make_rest_file_by_magic(
        "~/Ubuntu One/u1-test-magic-uploads-magic-2.txt",
        CONTENT, credentials)
    if success:
        print "Successful magic upload!"
    else:
        print ("Magic upload didn't succeed; you have to "
            "upload in the normal way.")

if __name__ == "__main__":
    main()