Uploading Documents and Setting Metadata Using SharePoint REST (One Version)

Technorati Tags: SP2013,ECM,REST

There are many examples of uploading documents using SharePoint 2013 REST/CSOM/JSOM and there are many issues. One issue is uploading documents into SharePoint Online with CSOM/JSOM. There is a 1.5mb limit. I work for a SharePoint ECM company and we have many customers that have documents much larger than 1.5mb. One way around this limitation is to use the SharePoint REST API which is limited  to 2gb. Just remember that REST requires you to post a byte array, and does not support reading from a stream. This can put a strain on memory resources.

Another issue is customers have versioning turned on and will complain that your solution creates two versions when uploading documents and setting metadata. This has always been a challenge when using SharePoint’s remote API. You can still use RPC which enables you to post the binary and the metadata in one call and only create one version. However, this can only be used from native apps and is limited to 50mb. You can upload and create only one version with CSOM/JSOM/REST by checking out the file before setting the metadata and then checking the file back in afterwards using the SPCheckinType.OverwriteCheckin. This works. However, if you try to check the file in and any field level validation fails, the check in fails. JavaScript code below.

function addFile() {   

    getFileBuffer().done(function (result) {
        upload(result.filename,result.content).done(function (data) {
            var file = data.d;
            checkOut(file.ServerRelativeUrl).done(function () {
                updateMetadata(file.ServerRelativeUrl, null).done(function () {
                    checkIn(file.ServerRelativeUrl).done(function () { });
                })
            })
            
        })
    }).fail(function (err) {
        var e = err;
    });
}

function getFileBuffer() {
    var file = $('#documentUpload')[0].files[0];
    var fileName = file.name;
    var dfd = $.Deferred();
    var reader = new FileReader();

    reader.onloadend = function (e) {
        var result = { 'filename': fileName, 'content': e.target.result };
        dfd.resolve(result);
    }
    reader.onerror = function (e) {
        dfd.reject(e.target.error);
    }

    reader.readAsArrayBuffer(file);
    return dfd;
}

function upload(filename, content) {
    appweburl = decodeURIComponent(getQueryStringParameter('SPAppWebUrl'));
    hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var restSource = appweburl +
        "/_api/SP.AppContextSite(@target)/web/lists/getbytitle('Documents')/rootfolder/files/add(url='" + filename + "',overwrite=true)?@target='" + hostweburl + "'";
    var dfd = $.Deferred();

    $.ajax(
       {
           'url': restSource,
           'method': 'POST',
           'data': content,
           processData: false,
           'headers': {
               'accept': 'application/json;odata=verbose',
               'X-RequestDigest': $('#__REQUESTDIGEST').val(),
               "content-length": content.byteLength
           },
           'success': function (data) {
               var d = data;
               dfd.resolve(d);
           },
           'error': function (err) {
               dfd.reject(err);
           }
       }
      );

    return dfd;
}
function checkOut(fileUrl) {
    appweburl = decodeURIComponent(getQueryStringParameter('SPAppWebUrl'));
    hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var restSource = appweburl +
        "/_api/SP.AppContextSite(@target)/web/lists/getbytitle('Documents')/rootfolder/files/getbyurl(url='" + fileUrl + "')/checkout?@target='" + hostweburl + "'";
    var dfd = $.Deferred();
    $.ajax(
    {
        'url': restSource,
        'method': 'POST',
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val()
        },
        'success': function (data) {
            var d = data;
            dfd.resolve(data.d);
        },
        'error': function (err) {
            dfd.reject(err);
        }
    }
   );

    return dfd;

}
function updateMetadata(fileUrl) {

    appweburl = decodeURIComponent(getQueryStringParameter('SPAppWebUrl'));
    hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var restSource = appweburl +
        "/_api/SP.AppContextSite(@target)/web/lists/getbytitle('Documents')/rootfolder/files/getbyurl(url='" + fileUrl + "')/listitemallfields?@target='" + hostweburl + "'";
    var dfd = $.Deferred();

    $.ajax(
    {
        'url': restSource,
        'method': 'POST',
        'data': JSON.stringify({
            '__metadata': {'type':'SP.ListItem'},
            'Title': 'My Title 3'
        }),
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val(),
            'X-Http-Method': 'PATCH',
            "If-Match": "*"
        },
        'success': function (data) {
            var d = data;
            dfd.resolve();
        },
        'error': function (err) {
            dfd.reject();
        }
    }
   );

    return dfd;

}
function checkIn(fileUrl) {
    appweburl = decodeURIComponent(getQueryStringParameter('SPAppWebUrl'));
    hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var restSource = appweburl +
        "/_api/SP.AppContextSite(@target)/web/lists/getbytitle('Documents')/rootfolder/files/getbyurl(url='" + fileUrl + "')/checkin?@target='" + hostweburl + "'";
    var dfd = $.Deferred();

    $.ajax(
    {
        'url': restSource,
        'method': 'POST',
        data: JSON.stringify({
            'checkInType': 2,
            'comment': 'whatever'
        }),
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val()
        },
        'success': function (data) {
            var d = data;
            dfd.resolve(data.d);
        },
        'error': function (err) {
            dfd.reject(err);
        }
    }
   );

    return dfd;

}

Use the ValidateUpdateListItem Method

If your going to be setting or updating metadata using the SharePoint’s remote API, then I suggest you use the SP.ListItem’s new ValidateUpdateListItem method. This method is available only through the remote API and is new to SP2013. ValidateUpdateListItem  is very similar to the UpdateOverwriteVersion method available on the server API. ValidateUpdateListItem sets the metdata and if the bNewDocumentUpdate argument is set to true will call the UpdateOverwriteVersion method which will update without incrementing the version. This eliminates the need to make the extra calls to check out and check in the document. It also will check in the document if it is already checked out and use the checkInComment argument.  The method takes multiple SP.ListItemFormUpdateValue types as arguments. This type takes the internal name of the field along with a value.

function updateMetadataNoVersion(fileUrl) {
    appweburl = decodeURIComponent(getQueryStringParameter('SPAppWebUrl'));
    hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var restSource = appweburl +
        "/_api/SP.AppContextSite(@target)/web/lists/getbytitle('Documents')/rootfolder/files/getbyurl(url='" + fileUrl+ "')/listitemallfields/validateupdatelistitem?@target='" + hostweburl + "'";

    var dfd = $.Deferred();

    $.ajax(
    {
        'url': restSource,
        'method': 'POST',
        'data': JSON.stringify({
            'formValues': [
                {
                '__metadata': { 'type': 'SP.ListItemFormUpdateValue' },
                'FieldName': 'Title',
                'FieldValue': 'My Title2'
            },
            {
                '__metadata': { 'type': 'SP.ListItemFormUpdateValue' },
                'FieldName': 'testautodate',
                'FieldValue': 'asdfsdfsdf'
            }
            ],
            'bNewDocumentUpdate': true,
            'checkInComment': ''
        }),
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val()
        },
        'success': function (data) {
            var d = data;
            dfd.resolve(d);
        },
        'error': function (err) {
            dfd.reject(err);
        }
    }
   );

    return dfd;
}

The efficiency of this method is reflected in its return type which is a list of SP.ListItemFormUpdateValue . This type contains the validation response for each field being updated. You can use the ListItemFormUpdateValue.HasException property to check for errors then use the ErrorMessage property to log or inform the user.

I Prefer ValidateUpdateListItem

Below is the revised code of adding a file using the ValidateUpdateListItem method.

function addFile() {   

    getFileBuffer().done(function (result) {
        upload(result.filename,result.content).done(function (data) {
            var file = data.d;
            updateMetadataNoVersion(file.ServerRelativeUrl).done(function () {
                
            })           
        })
    }).fail(function (err) {
        var e = err;
    });
}

The ValidateUpdateListItem method eliminates extra remote calls and allows you to handle multiple validation errors. Using this method along with REST you can efficiently upload a document up to 2gb and create only one version. You can also use this method to update metadata without having to create new file. Just set the bNewDocumentUpdate argument to true and this will not increment the version.

Using SharePoint 2013 REST API To Check User Permissions

Technorati Tags: SP2013,REST,Apps

Recently I have been working on a JavaScript app model project where I wanted to check the current user’s permissions within the site and on a particular list. As always I strive to use the SharePoint REST API when ever I can. So I looked at the the DoesUserHavePermissions method of the SPWeb. I used the SPRemoteAPIExplorer extension to look up this method to see what was needed to call it using REST. Version 2.0 now exposes complex types and the SP.BasePermissions type is used as an argument to this method. I wanted to check if the user had the ability to edit list items on the web. Looking at the generated ajax REST code from SPRemoteAPIExplorer I noticed it had two properties, High an Low. Since the SP.BasePermissions type is a flags type enumeration where you can combine permissions, these two properties represent the high order 32 bit integer and the low order 32 bit integer to a 64 bit integer representing the permission. The problem was how would I determine the high and low order of a given enumeration in JavaScript?

I wanted to avoid using JSOM and do everything with REST. Fortunately, I understand that you must rely on JSOM for certain things. In this case JSOM has the SP.BasePermissions type with methods for combining sets of permissions. This type is defined in SP.js. JSOM also exposes the types of permissions as a basic integer enumeration as SP.PermissionKind. This enumeration is defined in SP.Runtime.js.  I still could not figure out how to get the high an low values for the permission. I knew what the values were supposed to be for the EditLisitItems permission. Looking at the permission in debug view I noticed the values were exposed by the typical nonsensical property names $4_1 and $5_1 . When ever you set the permission with a permission kind the JSOM function will bit shift the values and re-calculate the high and low values.

The final thing to note is that even though the high and low values are 32 bit integers they must be sent as strings, otherwise you will get an error stating that it could not convert the primitive value to a EDM.Int64. The SharePoint REST processor expects these a strings when converting to a 64 bit integer. Why?  JavaScript does not support 64 bit integers and thus anything in a JSON payload would always be expected to be a string.  This is why when dealing with entities that represent 64 bit integers the JavaScript model will typically have a High and Low property. Mozilla has an example of something similar to SP.BasePermissions with its UInt64 type js-ctypes.

An example of a successful REST call to DoesUserHavePermissions :

function getUserWebPermissionREST() {
    var hostweburl = decodeURIComponent(getQueryStringParameter('SPHostUrl'));
    var appweburl = decodeURIComponent(getQueryStringParameter("SPAppWebUrl"));
    var restSource = appweburl + "/_api/web/doesuserhavepermissions";

    //still need jsom to get high order and low order numbers of a permission
    var perm = new SP.BasePermissions();
    perm.set(SP.PermissionKind.editListItems);

    $.ajax(
    {
        'url': restSource,
        'method': 'POST',
        'data': JSON.stringify({
            'permissionMask': {
                '__metadata': {
                    'type': 'SP.BasePermissions'
                },
                'High':  perm.$4_1.toString(),
                'Low': perm.$5_1.toString()
            }
        }),
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val()
        },
        'success': function (data) {
            var d = data.d.DoesUserHavePermissions;
        },
        'error': function (err) {
            alert(JSON.stringify(err));
        }
    }
   );
}

No matter how hard you try you still need to use JSOM

Trying to do everything using SharePoint REST is very difficult. When dealing with permissions it is much easier to leverage the built-in JSOM enumerations and SP.BasePermissions type rather than tying to replicate the same logic in your own libraries.  Also if your doing app model development you will need to use the SP.AppContextSite to get the context of the host web. So when doing app model development use REST as much as possible and use common sense when to use JSOM.  I hope this post helps you understand the reasoning for the SP.BasePermissions structure and its methods and why SharePoint REST uses a high and low property to represent a 64 bit integer for permissions. It would be nice if Microsoft would put actual friendly names around the $4_1 and $5_1 properties. 

Generate SharePoint 2013 REST Code with SPRemoteAPIExplorer 2.0

Technorati Tags: SP2013,Apps,Dev,VS2013,VS2012

One of the biggest road blocks for developers trying to learn the SharePoint 2013 REST API is the elusive complex type. Complex types are types that are used by the REST API to receive data as arguments from the client and to send data back to the client as a response. As a SharePoint developer you know and “love” these  types by there JSON signature of {‘_metadata’:{type: SP.Whatever},…}. Having to figure these type names and what properties are available is a constant struggle. The complex type are not the same as entity types returned from libraries, lists and feeds but are typically used in API calls to do such things are create sub-sites or social posts. Complex types are named appropriately because they can be complex.

Here are some quick facts about complex types. There are 154 complex types defined in the REST API not all of which can be used by REST.  There are 22 complex types which contain nested complex types. For example, the SP.Social.SocialRestPostCreationData complex type used to create a social post contains 5 nested complex types.  There are 7 complex collection types which contain complex child item types. These collection complex types have a different JSON signature than complex types.  Finally of the 154 complex types 68 are marked as either internal or private making them impossible to discover without the use of a reflection tool. The new $metadata schema information available in SP1 for the REST API is useful but does not give you any clues on how to construct the complex type in a POST. All of these facts make the SharePoint 2013 REST API difficult to learn and use.

SPRemoteAPIExplorer -- Visual Studio Gallery

SPRemoteAPIExplorer 2.0 Supports Discovery of REST Complex Types

In the previous version of SPRemoteAPIExplorer you could not see what properties were contained in the complex type if it was being used as a parameter to a method call. With version 2.0 you can now see complex types marked with a different icon in the explorer. You can expand it and see the properties contained within. The same information available for non-complex parameters is available such as whether it can be used in CSOM, JSOM or Managed Code.

As you can see the this complex type has many nested complex types. So how can you easily create this type in JSON and post it in the REST call?

Generate REST calls with SPRemoteAPIExplorer 2.0

Version 2.0 supports right clicking on the method and selecting the “Create $ajax call” menu item.

This will create a jQuery $.ajax code snippet and copy the code to the clipboard. The code will contain the JSON payload definition for the post setting the values to null. There is no need to lookup the type names and manage all the nesting. Also SPRemoteExplorer will also add proper syntax for multi- value properties and collection complex types. Below is the code that is generated. You can paste this into your javascript and start testing the call.

 ///method: Post resource: SP.Social.SocialRestFeed
    $.ajax(
    {
        'url': 'restSource',
        'method': 'POST',
        'data': JSON.stringify({
            'restCreationData': {
                '__metadata': {
                    'type': 'SP.Social.SocialRestPostCreationData'
                },
                'ID': null,
                'creationData': {
                    '__metadata': {
                        'type': 'SP.Social.SocialPostCreationData'
                    },
                    'Attachment': {
                        '__metadata': {
                            'type': 'SP.Social.SocialAttachment'
                        },
                        'AttachmentKind': null,
                        'ClickAction': {
                            '__metadata': {
                                'type': 'SP.Social.SocialAttachmentAction'
                            },
                            'ActionKind': null,
                            'ActionUri': null
                        },
                        'ContentUri': null,
                        'Description': null,
                        'Height': null,
                        'Length': null,
                        'Name': null,
                        'PreviewUri': null,
                        'Uri': null,
                        'Width': null
                    },
                    'ContentItems': { 'results': [''] },
                    'ContentText': null,
                    'DefinitionData': {
                        '__metadata': {
                            'type': 'SP.Social.SocialPostDefinitionData'
                        },
                        'Items': { 'results': [''] },
                        'Name': null
                    },
                    'SecurityUris': { 'results': [''] },
                    'Source': {
                        '__metadata': {
                            'type': 'SP.Social.SocialLink'
                        },
                        'Text': null,
                        'Uri': null
                    },
                    'UpdateStatusText': null
                }
            }
        }),
        'headers': {
            'accept': 'application/json;odata=verbose',
            'content-type': 'application/json;odata=verbose',
            'X-RequestDigest': $('#__REQUESTDIGEST').val()
        },
        'success': function (data) {
            var d = data;
        },
        'error': function (err) {
            alert(JSON.stringify(err));
        }
    }
   );

Sometimes you may only want to get the complex type JSON representation. This can be accomplished by right clicking on a complex type node and selecting the “Create JSON” menu item.  This will copy the JSON to the clipboard.

No more searching for SharePoint 2013 REST code samples

SPRemoteAPIExplorer 2.0 should save you a lot of time searching for code samples on how to call many of the undocumented REST methods. The new code generation feature should make it easier to test the capabilities of the API. It will give you a chance to test this tool out also. I purposely allowed the REST code to be generated on methods that are marked as not supporting REST. This way you can test the validity  of this assertion. You will find that even though the method is marked as supporting REST, if one of the parameter’s “OData Type” property is marked as Invalid, then the REST call will fail.  I have developed this tool and SPFastDeploy to make SharePoint Online development quicker and easier. I hope you find some usefulness from this tool and please feel free to suggest improvements either here or on the Visual Studio Gallery Q/A section.