From 1e61357680055dd5743846768ab923569772f6aa Mon Sep 17 00:00:00 2001 From: David Stern Date: Wed, 2 Mar 2022 16:23:57 +0000 Subject: [PATCH] Add capability to get and set attributes in compute contexts (#97) * New updatecomputecontext.py & mods to callrestapi * Rename to setcomputecontextattributes.py * Updates to get and setcomputecontextattributes.py * Example, and exception handlers for python3 * tidy exception handler up * Don't pop items fr dict while we iterate over them * Cut stuff we don't need to make dbg output clearer * Minor tweaks after debugging * Set executable bit on new scripts --- EXAMPLES.md | 13 ++ getcomputecontextattributes.py | 99 +++++++++++++ ...ntext.py => setcomputecontextattributes.py | 133 +++++------------- sharedfunctions.py | 27 +++- 4 files changed, 172 insertions(+), 100 deletions(-) create mode 100755 getcomputecontextattributes.py rename updatecomputecontext.py => setcomputecontextattributes.py (67%) mode change 100644 => 100755 diff --git a/EXAMPLES.md b/EXAMPLES.md index 2cf9c36..8352cd3 100644 --- a/EXAMPLES.md +++ b/EXAMPLES.md @@ -342,3 +342,16 @@ For example: ./submit_jobreq.py -id {jobRequestId} ./submit_jobreq.py -id {jobRequestId} -v ``` + +**getcomputecontextattributes.py** + +```bash +./getcomputecontextattributes.py -n "Data Mining compute context" +``` + +**setcomputecontextattributes.py** + +```bash +./setcomputecontextattributes.py -n "Data Mining compute context" -a runAsUser -v sastest1 +./setcomputecontextattributes.py -n "Data Mining compute context" -r runAsUser +``` \ No newline at end of file diff --git a/getcomputecontextattributes.py b/getcomputecontextattributes.py new file mode 100755 index 0000000..0ef796e --- /dev/null +++ b/getcomputecontextattributes.py @@ -0,0 +1,99 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# getcomputecontextattributes.py +# February 2022 +# +# Print compute context attributes. +# +# +# Change History +# +# Copyright © 2022, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +# OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. +# +# +# Import Python modules +import argparse, sys, os, json +from sharedfunctions import callrestapi + +debug=False + +# Define exception handler so that we only output trace info from errors when in debug mode +def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook): + if debug: + debug_hook(exception_type, exception, traceback) + else: + print (exception_type.__name__, exception) + +sys.excepthook = exception_handler + +# get input parameters +parser = argparse.ArgumentParser(description="List attributes in an existing compute context. If you want to add, remove or update a compute context, use setcomputecontextattributes.py.") +parser.add_argument("-n","--name", help="Compute context name",required='True') +args= parser.parse_args() +contextname=args.name + +# get python version +#version=int(str(sys.version_info[0])) +#print("Python version: " + str(version)) + +# Get compute contexts +reqtype="get" +reqval="/compute/contexts/?filter=eq(name, '"+contextname+"')" +resultdata=callrestapi(reqval,reqtype) +#json_formatted_str = json.dumps(resultdata, indent=2) +#print(json_formatted_str) + + +if 'items' in resultdata: + #print(resultdata['items']) + if resultdata['items']==[]: + id=None + raise Exception("Compute context '"+contextname+"' not found.") + elif len(resultdata['items'])>1: + id=None + raise Exception("More than one matching compute context named '"+contextname+"'.!") + # If we make it this far, we found exactly one compute context + for i in resultdata['items']: + id=i['id'] + #print("Compute context: "+contextname+" ["+id+"]") +else: + id=None + # Handle the error! Compute context not found... + raise Exception('Compute context not found.') + +if id!=None: + + # Get the details of this compute context + reqtype="get" + reqval="/compute/contexts/"+id + resultdata,etag=callrestapi(reqval,reqtype,returnEtag=True) + #print("etag: "+etag) + # Get rid of parts of the context we don't need + resultdata.pop("links",None) + resultdata.pop("creationTimeStamp",None) + resultdata.pop("modifiedTimeStamp",None) + resultdata.pop("version",None) + #json_formatted_str = json.dumps(resultdata, indent=2) + #print(json_formatted_str) + + boolFoundAttribute=False + if 'attributes' in resultdata: + # numattributes=len(resultdata['attributes']) + # print('numattributes='+str(numattributes)) + # print(resultdata['attributes']) + # print(type(resultdata['attributes'])) + + for attributeKey, attributeValue in resultdata['attributes'].items(): + print("Attribute: "+attributeKey+" : "+attributeValue) + else: + # No attributes section in results data at all + print("Compute context '"+contextname+"' has no attributes") + +sys.exit() \ No newline at end of file diff --git a/updatecomputecontext.py b/setcomputecontextattributes.py old mode 100644 new mode 100755 similarity index 67% rename from updatecomputecontext.py rename to setcomputecontextattributes.py index 9daba12..8cec7b9 --- a/updatecomputecontext.py +++ b/setcomputecontextattributes.py @@ -19,9 +19,8 @@ # # # Import Python modules -import argparse, sys, os, json -from sharedfunctions import callrestapi, getbaseurl, getauthtoken -import requests +import argparse, sys, json +from sharedfunctions import callrestapi debug=False @@ -30,82 +29,15 @@ def exception_handler(exception_type, exception, traceback, debug_hook=sys.excep if debug: debug_hook(exception_type, exception, traceback) else: - print "%s: %s" % (exception_type.__name__, exception) + print (exception_type.__name__, exception) sys.excepthook = exception_handler - - -########################################################### -def callrestapiwithetag(reqval, reqtype, acceptType='application/json', contentType='application/json',etagIn='',data={},stoponerror=1): - - # get the url from the default profile - baseurl=getbaseurl() - - # get the auth token - oaval=getauthtoken(baseurl) - - # build the authorization header - head= {'Content-type':contentType,'Accept':acceptType} - head.update({"Authorization" : oaval}) - if etagIn!='': - head.update({"If-Match" : etagIn}) - - # maybe this can be removed - global result - - # sereliaze the data string for the request to json format - json_data=json.dumps(data, ensure_ascii=False) - - # call the rest api using the parameters passed in and the requests python library - - if reqtype=="get": - ret = requests.get(baseurl+reqval,headers=head,data=json_data) - elif reqtype=="post": - ret = requests.post(baseurl+reqval,headers=head,data=json_data) - elif reqtype=="delete": - ret = requests.delete(baseurl+reqval,headers=head,data=json_data) - elif reqtype=="put": - ret = requests.put(baseurl+reqval,headers=head,data=json_data) - else: - result=None - print("NOTE: Invalid method") - sys.exit() - - - # response error if status code between these numbers - if (400 <= ret.status_code <=599): - - print(ret.text) - result=None - if stoponerror: sys.exit() - - # return the result - else: - # is it json - try: - result=ret.json() - except: - # is it text - try: - result=ret.text - except: - result=None - print("NOTE: No result to print") - - etagOut=None - if 'etag' in ret.headers: - etagOut=ret.headers['etag'] - - return result,etagOut; - -########################################################### - # get input parameters parser = argparse.ArgumentParser(description="Add attributes to an existing compute context.") parser.add_argument("-n","--name", help="Compute context name",required='True') parser.add_argument("-a","--add", help="Single attribute to add or update.", nargs="?",const="") -parser.add_argument("-v","--value", help="Value to set attribute to. Only has any effect if you also specify an attribute to add.", nargs="?",const="") +parser.add_argument("-v","--value", help="Value to set attribute to. Only has any effect if the arguments also specify an attribute to add or update with -a.", nargs="?",const="") parser.add_argument("-r","--remove", help="Single attribute to remove. If the compute context does not have this attribute, nothing happens.", nargs="?",const="") args= parser.parse_args() contextname=args.name @@ -115,16 +47,16 @@ attrToRemove=str(args.remove) if attrToAdd != "None": if attrValue == "None": - raise Exception('If you specify an attribute to add or update, you must also specify a value using -v or --value.') + raise Exception('If the arguments specify an attribute to add or update, they must also specify a value using -v or --value.') if attrToRemove != "None": - raise Exception('If you specify an attribute to add or update, do not also specify an attribute to remove in the same command. Add and remove attributes with separate calls to this utility.') + raise Exception('If the arguments specify an attribute to add or update, they should not also specify an attribute to remove in the same command. Add and remove attributes with separate calls to this utility.') else: if attrToRemove == "None": - raise Exception('You must specify either:\n - an attribute to add or update with -a and a value to set it to with -v, or\n - an attribute to remove with -r.') + raise Exception('The arguments specify either:\n - an attribute to add or update with -a and a value to set it to with -v, or\n - an attribute to remove with -r.') if attrToRemove != "None": if attrValue != "None": - print('Note: You specified an attribute to remove, but also specified a value. The value will be ignored; the specified attribute will be removed whatever value it has.') + print('Note: Arguments specify an attribute to remove, and also specified a value. The value will be ignored; the specified attribute will be removed whatever value it has.') # get python version #version=int(str(sys.version_info[0])) @@ -164,14 +96,16 @@ if id!=None: reqval="/compute/contexts/"+id # reqaccept="application/vnd.sas.compute.context.summary+json" # reccontent="application/vnd.sas.collection+json" - resultdata,etag=callrestapiwithetag(reqval,reqtype) - #print(etag) - # Get rid of parts of the context we don't need + resultdata,etag=callrestapi(reqval,reqtype,returnEtag=True) + #print("etag: "+etag) + + # Get rid of parts of the context structure that are not required for updating the context resultdata.pop("links",None) resultdata.pop("creationTimeStamp",None) resultdata.pop("modifiedTimeStamp",None) resultdata.pop("version",None) - json_formatted_str = json.dumps(resultdata, indent=2) + + #json_formatted_str = json.dumps(resultdata, indent=2) #print(json_formatted_str) # The following set of logic expects and assumes that EITHER: @@ -188,8 +122,10 @@ if id!=None: # print(resultdata['attributes']) # print(type(resultdata['attributes'])) + boolRemoveAfterIterating=False + for attributeKey, attributeValue in resultdata['attributes'].items(): - print("Attribute: "+attributeKey+" : "+attributeValue) + #print("Attribute: "+attributeKey+" : "+attributeValue) if attrToAdd != "None": # We are adding or updating a value @@ -210,14 +146,9 @@ if id!=None: # We found the value to remove boolFoundAttribute=True print("Attribute: "+attributeKey+" : "+attributeValue+" to be removed") - if len(resultdata['attributes'].items())==1: - # We are about to remove the only attribute - # Remove the whole attributes dictionary - resultdata.pop("attributes",None) - else: - # We are about to remove an attribute, but it is not the last one - # Remove this specific attribute from the 'attributes' dictionary - resultdata['attributes'].pop(attrToRemove,None) + # However, don't remove it yet, as we are still iterating over the attributes + # and Python 3 doesn't like you deleting an element of the dict you are iterating over. + boolRemoveAfterIterating=True boolUpdateRequired=True if not boolFoundAttribute: if attrToAdd != "None": @@ -228,6 +159,19 @@ if id!=None: if attrToRemove != "None": # We are being asked to remove an attribute, but we did not find it among the compute context's existing attributes print("Attribute: "+attrToRemove+" was not found and cannot be removed") + + if boolRemoveAfterIterating: + # Now we are no longer iterating over the attributes list, so it's okay to delete either + # an element from the list, or the entire list + if len(resultdata['attributes'].items())==1: + # We are about to remove the only attribute + # Remove the whole attributes dictionary + resultdata.pop("attributes",None) + else: + # We are about to remove an attribute, but it is not the last one + # Remove this specific attribute from the 'attributes' dictionary + resultdata['attributes'].pop(attrToRemove,None) + else: # No attributes section in results data at all if attrToAdd != "None": @@ -245,7 +189,6 @@ if id!=None: #print(json_formatted_str) if boolUpdateRequired: - print("Update required") # Update compute contexts # See http://swagger.na.sas.com/swagger-ui/?url=/apis/compute/v10/openapi-all.json#/Contexts/updateContext ########################################################################## @@ -260,16 +203,16 @@ if id!=None: # response header of any endpoint that produces # application/vnd.sas.compute.context. ########################################################################## + #print("Update required") reqtype="put" reqval="/compute/contexts/"+id reqaccept="application/vnd.sas.compute.context+json" - #reccontent="application/vnd.sas.collection+json" reccontent="application/vnd.sas.compute.context+json" - resultdata_after_update,etagAfter=callrestapiwithetag(reqval,reqtype,reqaccept,reccontent,etag,data=resultdata) - json_formatted_str = json.dumps(resultdata_after_update, indent=2) + resultdata_after_update=callrestapi(reqval,reqtype,reqaccept,reccontent,data=resultdata,stoponerror=False,etagIn=etag) + #json_formatted_str = json.dumps(resultdata_after_update, indent=2) #print(json_formatted_str) - else: - print("Update not required") + #else: + #print("Update not required") sys.exit() diff --git a/sharedfunctions.py b/sharedfunctions.py index 8c5c79d..3dec907 100755 --- a/sharedfunctions.py +++ b/sharedfunctions.py @@ -36,6 +36,7 @@ # 09dec2020 Added get_valid_filename function to deal with invalid characters for Linux filesystem # 16Jul2021 Edited callrestapi to be able to update the header. (Issue #83) # 20Feb2022 Support patch +# 28Feb2022 Added functionality to callrestapi optionally pass in etags, and to request they be returned, for API endpoints that use them # # Copyright © 2018, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # @@ -91,8 +92,10 @@ def validaterestapi(baseurl, reqval, reqtype, data={}): # 01dec2017 initial development # 28oct2018 Added stop on error to be able to override stopping processing when an error occurs # 16Jul2021 Added a functionality to update the header if necessary. +# 20Feb2022 Support patch +# 28Feb2022 Added functionality to optionally pass in etags, and to request they be returned, for API endpoints that use them -def callrestapi(reqval, reqtype, acceptType='application/json', contentType='application/json',data={},header={},stoponerror=1): +def callrestapi(reqval, reqtype, acceptType='application/json', contentType='application/json',data={},header={},stoponerror=1,returnEtag=False,etagIn=''): # get the url from the default profile @@ -107,11 +110,14 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app #Converting the header values to string to pass it into the header header = {str(key):str(value) for key,value in header.items()} head.update(header) + # If an etag was passed in, add an If-Match header with that etag as the value + if etagIn!='': + head.update({"If-Match" : etagIn}) # maybe this can be removed global result - # sereliaze the data string for the request to json format + # serialize the data string for the request to json format json_data=json.dumps(data, ensure_ascii=False) # call the rest api using the parameters passed in and the requests python library @@ -135,7 +141,8 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app # response error if status code between these numbers if (400 <= ret.status_code <=599): - print(ret.text) + print("http response code: "+ str(ret.status_code)) + print("ret.text: "+ret.text) result=None if stoponerror: sys.exit() @@ -152,9 +159,19 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app result=None print("NOTE: No result to print") + # Capture the value of any etag returned in the headers + etagOut=None + if 'etag' in ret.headers: + etagOut=ret.headers['etag'] - - return result; + # ONLY if the caller specifically asked for an etag to be returned, return one + if returnEtag: + return result,etagOut; + else: + # Otherwise, return only the result as normal. + # This avoids breaking anything that does not expect an etag to be returned + # in addition to the normal results. + return result; # getfolderid # when a Viya content path is passed in return the id, path and uri