From 4de26a53983b45af843dd756f616e045d1d2f2b7 Mon Sep 17 00:00:00 2001 From: Gerry Nelson <45608078+gerrynelson63@users.noreply.github.com> Date: Wed, 23 Dec 2020 11:49:21 -0500 Subject: [PATCH] Viya4 v1 (#64) * add properties file * add get properties function * add get of properties to listreports * update listreports * fix * add insepct * test with foldertree * update properties * test export folder tree * exportfoldertree * make cli location and command dynamic * add imports * fix * add os * add all option * updates * update import packages * update import packages * a few more sas-viya changes * add function * showsetup * make targetname optional * update * update * Update sharedfunctions.py added get_valid_filename * Update snapshotreports.py * Update updatepreferences.py upped the limit on all users * Update INSTALL.md * Update INSTALL.md --- INSTALL.md | 18 ++ application.properties | 2 + applyfolderauthorization.py | 31 +-- createbinarybackup.py | 17 +- explainaccess.py | 42 ++-- exportfoldertree.py | 47 +++-- getpath.py | 12 +- importpackages.py | 23 +- listreports.py | 5 +- loginviauthinfo.py | 29 +-- sharedfunctions.py | 405 +++++++++++++++++++----------------- showsetup.py | 20 +- snapshotreports.py | 23 +- testfolderaccess.py | 20 +- updatepreferences.py | 75 ++++--- 15 files changed, 450 insertions(+), 319 deletions(-) create mode 100644 application.properties mode change 100644 => 100755 listreports.py diff --git a/INSTALL.md b/INSTALL.md index ec30590..38cf4a2 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -10,6 +10,24 @@ The tools should be installed on the same machine that hosts the Viya command-li *git clone https://github.com/sassoftware/pyviyatools.git* +NOTE: to use the tools with Viya 4 clone the viya4_v1 branch + +*git clone https://github.com/sassoftware/pyviyatools.git -b viya4_v1* + +**Configure** + +The application.properties file contains the default location of the sas-admin or viya cli. Edit this file to match the location of the cli and the cli name in your environment. + +The default values for Viya 3.x are: + +sascli.location=/opt/sas/viya/home/bin/ +sascli.executable=sas-admin + +The default values for Viya 4.x are: + +sascli.location=/opt/sas/viya/home/bin/ +sascli.executable=sas-viya + **Authenticate** The pyviya tools use the sas-admin auth CLI to authenticate to Viya. To use the tool you must create a profile and authenticate. This process is documented in the SAS Viya Administration guide here. diff --git a/application.properties b/application.properties new file mode 100644 index 0000000..530214d --- /dev/null +++ b/application.properties @@ -0,0 +1,2 @@ +sascli.location=/opt/sas/viya/home/bin/ +sascli.executable=sas-viya \ No newline at end of file diff --git a/applyfolderauthorization.py b/applyfolderauthorization.py index ad2bf76..f45977e 100755 --- a/applyfolderauthorization.py +++ b/applyfolderauthorization.py @@ -10,14 +10,14 @@ # # Format of input csv file is 6 columns # Column 1 is the full path to the folder -# Column 2 is the principal type +# Column 2 is the principal type # Column 3 is the principal name # Column 4 is the access setting (grant or prohibit) # Column 5 is the permissions on the folder -# Column 6 is the conveyed permissions on the folder's contents +# Column 6 is the conveyed permissions on the folder's contents # # For example: -# /gelcontent/gelcorp/marketing/reports,group,Marketing,grant,"read,add,remove","read,update,add,remove,delete,secure" +# /gelcontent/gelcorp/marketing/reports,group,Marketing,grant,"read,add,remove","read,update,add,remove,delete,secure" # # Copyright 2020, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # @@ -38,14 +38,19 @@ import csv import os import json import subprocess -from sharedfunctions import callrestapi, getfolderid, file_accessible, printresult +import sys +from sharedfunctions import callrestapi, getfolderid, file_accessible, printresult,getapplicationproperties -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' -#clidir='c:\\admincli\\' +# get cli location from properties +propertylist=getapplicationproperties() +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] -# setup command-line arguements +clicommand=os.path.join(clidir,cliexe) + + +# setup command-line arguements parser = argparse.ArgumentParser(description="Apply bulk auths from a CSV file to folders and contents") parser.add_argument("-f","--file", help="Full path to CSV file. Format of csv: 'folderpath,principaltype,principalid,grant_or_prohibit,perms_on_folder,perms_on_contents",required='True') args = parser.parse_args() @@ -74,14 +79,14 @@ if check: folderid=getfolderid(folderpath) folderuri=folderid[0] reqval='/folders/folders/'+folderuri - + # Construct JSON objects from auth rules defined in CSV. Two JSON objects are created for each row of CSV; one for perms on the folder object, one for conveyed perms on the object's contents. value_dict_object={"description":"Created by applyfolderauthorizations.py", "objectUri":reqval, "permissions":folderpermissions.split(','), "principalType":principaltype, "principal":principalname, - "type":accesssetting + "type":accesssetting } value_dict_container={"description":"Created by applyfolderauthorizations.py", "containerUri":reqval, @@ -101,17 +106,17 @@ if check: } constructed_bulk_rules_list.append(constructed_rule_dict_object) constructed_bulk_rules_list.append(constructed_rule_dict_container) - + else: print("ERROR: cannot read "+file) print("Writing out bulk rule JSON file to bulk_rules_list.json") -# Construct JSON schema containing rules +# Construct JSON schema containing rules bulk_rules_list_string=json.dumps(constructed_bulk_rules_list,indent=2) with open("bulk_rules_list.json", "w") as text_file: text_file.write(bulk_rules_list_string+'\n') # Execute sas-admin CLI to apply rules from JSON schema -command=clidir+'sas-admin authorization create-rules --file bulk_rules_list.json' +command=clicommand+' authorization create-rules --file bulk_rules_list.json' print("Executing command: "+command) subprocess.call(command, shell=True) diff --git a/createbinarybackup.py b/createbinarybackup.py index 6ef1eef..ad96448 100755 --- a/createbinarybackup.py +++ b/createbinarybackup.py @@ -23,8 +23,19 @@ # limitations under the License. # -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' +# get python version +version=int(str(sys.version_info[0])) + +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) + + + debug=False defaultBackupScheduleName="DEFAULT_BACKUP_SCHEDULE" newScheduleName="BINARY_BACKUP_SCHEDULE" @@ -104,7 +115,7 @@ if debug: print('jobExecutionRequest_json:') print(jobExecutionRequest_json) -# STEP 3 of 4: Get the href to submit the job from the create jobExecution response +# STEP 3 of 4: Get the href to submit the job from the create jobExecution response links=jobExecutionRequest_json['links'] href_found=False diff --git a/explainaccess.py b/explainaccess.py index c106a75..51b60d1 100755 --- a/explainaccess.py +++ b/explainaccess.py @@ -21,7 +21,7 @@ # 4. As 1. with a header row and the folder path, which is useful if you concatenate sets of results in one file # ./explainaccess.py -f /folderA/folderB -p --header # -# 5. As 1. showing only rows which include a direct grant or prohibit +# 5. As 1. showing only rows which include a direct grant or prohibit # ./explainaccess.py -f /folderA/folderB --direct_only # # 6. Explain direct and indirect permissions on a service endpoint. Note in the results that there are no conveyed permissions. @@ -35,7 +35,7 @@ # since none of add, remove or create are applicable to a report. # ./explainaccess.py -u /reports/reports/e2e0e601-b5a9-4601-829a-c5137f7441c6 --header -l read update delete secure # -# 9. Explain direct and indirect permissions on a folder expressed as a URI. Keep the default permissions list, but for completeness +# 9. Explain direct and indirect permissions on a folder expressed as a URI. Keep the default permissions list, but for completeness # we must also specify -c true to request conveyed permissions be displayed, as they are not displayed by default for URIs. # ./explainaccess.py -u /folders/folders/9145d26a-2c0d-4523-8835-ad186bb57fa6 --header -p -c true # @@ -54,8 +54,25 @@ # limitations under the License. # +# Import Python modules + +import argparse +import subprocess +import json +import sys +from sharedfunctions import getfolderid,callrestapi,getapplicationproperties + +# get python version +version=int(str(sys.version_info[0])) + +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) -clidir='/opt/sas/viya/home/bin/' debug=False direct_only=False valid_permissions=['read','update','delete','secure','add','remove','create'] @@ -64,15 +81,6 @@ default_permissions=['read','update','delete','secure','add','remove'] direct_permission_suffix='*' -# Import Python modules - -import argparse -import subprocess -import json -import sys - -from sharedfunctions import getfolderid,callrestapi - # Define exception handler so that we only output trace info from errors when in debug mode def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook): if debug: @@ -119,7 +127,7 @@ for permission in permissions: if permission not in valid_permissions: raise Exception(permission+' is not the name of a permission. Valid permissions are: '+' '.join(map(str, valid_permissions))) -# Two ways this program can be used: for a folder, or for a URI. +# Two ways this program can be used: for a folder, or for a URI. if path_to_folder: getfolderid_result_json=getfolderid(path_to_folder) @@ -141,7 +149,7 @@ if path_to_folder: convey=False else: convey=True - + else: explainuri=objecturi # This tool explains the permissions of any object. @@ -161,7 +169,7 @@ else: else: convey=False - + #Use the /authorization/decision endpoint to ask for an explanation of the rules that are relevant to principals on this URI #See Authorization API documentation in swagger at http://swagger.na.sas.com/apis/authorization/v4/apidoc.html#op:createExplanation endpoint='/authorization/decision' @@ -215,7 +223,7 @@ for pi in e: # Permissions on object for permission in permissions: # Not all objects have all the permissions - # Note that some objects do have permissions which are not meaningful for that object. + # Note that some objects do have permissions which are not meaningful for that object. # E.g. SASAdministrators are granted Add and Remove on reports, by an OOTB rule which grants SASAdministrators all permissions (including Add and Remove) on /**. # Meanwhile, Add and Remove are not shown in the View or Edit Authotizations dialogs for reports in EV etc. # So, while it may be correct for the /authorization/decisions endpoint to explain that SASAdministrators are granted Add and Remove on a report, @@ -271,7 +279,7 @@ for pi in e: else: # This permission was absent from the expanation for this principal outstr=outstr+',' - # Conveyed permissions + # Conveyed permissions if convey: for permission in permissions: # Not all objects have all the permissions diff --git a/exportfoldertree.py b/exportfoldertree.py index 80ffb92..65213ce 100755 --- a/exportfoldertree.py +++ b/exportfoldertree.py @@ -7,7 +7,7 @@ # Pass in a directory and this tool will export the complete viya folder # structure to a sub-directory named for the current date and time # There will be a json file for each viya folder at the root level. -# +# # # Change History # @@ -30,15 +30,20 @@ # Import Python modules import argparse, sys, subprocess, uuid, time, os, glob -from sharedfunctions import getfolderid, callrestapi +from sharedfunctions import getfolderid, callrestapi,getapplicationproperties -# get python version +# get python version version=int(str(sys.version_info[0])) -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) -# get input parameters +# get input parameters parser = argparse.ArgumentParser(description="Export the complete Viya folder tree") parser.add_argument("-d","--directory", help="Directory for Export",required='True') parser.add_argument("-q","--quiet", help="Suppress the are you sure prompt.", action='store_true') @@ -56,11 +61,11 @@ if os.path.exists(basedir): if version > 2: areyousure=input("The folder exists any existing json files in it will be deleted. Continue? (Y)") else: - areyousure=raw_input("The folder already exists any existing json files in it will be deleted. Continue? (Y)") + areyousure=raw_input("The folder already exists any existing json files in it will be deleted. Continue? (Y)") else: areyousure="Y" -else: areyousure="Y" +else: areyousure="Y" # prompt is Y if user selected Y, its a new directory, or user selected quiet mode if areyousure.upper() =='Y': @@ -69,7 +74,7 @@ if areyousure.upper() =='Y': # create directory if it doesn't exist if not os.path.exists(path): os.makedirs(path) - else: + else: filelist=glob.glob(path+"/*.json") for file in filelist: os.remove(file) @@ -81,31 +86,31 @@ if areyousure.upper() =='Y': # loop root folders if 'items' in resultdata: - + total_items=resultdata['count'] - + returned_items=len(resultdata['items']) - + if total_items == 0: print("Note: No items returned.") else: # export each folder and download the package file to the directory - for i in range(0,returned_items): - + for i in range(0,returned_items): + id=resultdata['items'][i]["id"] package_name=str(uuid.uuid1()) json_name=resultdata['items'][i]["name"].replace(" ","")+'_'+str(i) - - command=clidir+'sas-admin transfer export -u /folders/folders/'+id+' --name "'+package_name+'"' - print(command) + + command=clicommand+' transfer export -u /folders/folders/'+id+' --name "'+package_name+'"' + print(command) subprocess.call(command, shell=True) - reqval='/transfer/packages?filter=eq(name,"'+package_name+'")' + reqval='/transfer/packages?filter=eq(name,"'+package_name+'")' package_info=callrestapi(reqval,reqtype) - + package_id=package_info['items'][0]['id'] - + completefile=os.path.join(path,json_name+'.json') - command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id + command=clicommand+' transfer download --file '+completefile+' --id '+package_id print(command) subprocess.call(command, shell=True) diff --git a/getpath.py b/getpath.py index b04f474..ae3fe53 100755 --- a/getpath.py +++ b/getpath.py @@ -31,7 +31,17 @@ # -clidir='/opt/sas/viya/home/bin/' +# get python version +version=int(str(sys.version_info[0])) + +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) + debug=False # Import Python modules diff --git a/importpackages.py b/importpackages.py index 91e4ab7..f857c64 100755 --- a/importpackages.py +++ b/importpackages.py @@ -15,18 +15,23 @@ # Licensed under the Apache License, Version 2.0 (the License); you may not use this file except in compliance with the License. # You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS +# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS # OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. # # change log # renamed to importpackages.py to be more descriptive of actual usage # # Import Python modules -import argparse, sys, subprocess, os, json -from sharedfunctions import callrestapi +import argparse, sys, subprocess, os, json +from sharedfunctions import callrestapi, getapplicationproperties -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) # get input parameters parser = argparse.ArgumentParser(description="Import JSON files from directory. All json files in directory will be imported.") @@ -36,7 +41,7 @@ args= parser.parse_args() basedir=args.directory quietmode=args.quiet -# get python version +# get python version version=int(str(sys.version_info[0])) @@ -46,7 +51,7 @@ if not quietmode: if version > 2: areyousure=input("WARNING: If content from the packages already exists in folders it will be replaced. Continue? (Y)") else: - areyousure=raw_input("WARNING:If content from the packages already exists in folders it will be replaced. Continue? (Y)") + areyousure=raw_input("WARNING:If content from the packages already exists in folders it will be replaced. Continue? (Y)") else: areyousure="Y" @@ -62,7 +67,7 @@ if areyousure.upper() =='Y': if filename.lower().endswith('.json'): #upload the json package - command=clidir+'sas-admin transfer upload --file '+os.path.join(basedir,filename)+'> /tmp/packageid.json' + command=clicommand+' transfer upload --file '+os.path.join(basedir,filename)+'> /tmp/packageid.json' print(command) subprocess.call(command, shell=True) @@ -74,7 +79,7 @@ if areyousure.upper() =='Y': # get the packageid and import the package packageid=package_data["id"] - command=clidir+'sas-admin --output text -q transfer import --id '+packageid + command=clicommand+' --output text -q transfer import --id '+packageid print(command) subprocess.call(command, shell=True) diff --git a/listreports.py b/listreports.py old mode 100644 new mode 100755 index 051c3f2..3c858a9 --- a/listreports.py +++ b/listreports.py @@ -27,14 +27,11 @@ # Import Python modules import argparse, sys, subprocess, uuid, time, os, glob from datetime import datetime as dt, timedelta as td -from sharedfunctions import getfolderid, callrestapi, getpath, printresult +from sharedfunctions import getfolderid, callrestapi, getpath, printresult, getapplicationproperties # get python version version=int(str(sys.version_info[0])) -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' - # get input parameters parser = argparse.ArgumentParser(description="List Viya Reports and their folder path.") parser.add_argument("-n","--name", help="Name contains?",default=None) diff --git a/loginviauthinfo.py b/loginviauthinfo.py index 916c17f..cf7466a 100755 --- a/loginviauthinfo.py +++ b/loginviauthinfo.py @@ -47,7 +47,7 @@ import os import argparse import json -from sharedfunctions import file_accessible +from sharedfunctions import file_accessible,getapplicationproperties try: # Python 3 @@ -57,14 +57,19 @@ except ImportError: from urlparse import urlparse -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' -#clidir='c:\\admincli\\' +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) + debug=0 profileexists=0 -# get input parameters +# get input parameters parser = argparse.ArgumentParser(description="Authinfo File") parser.add_argument("-f","--file", help="Enter the path to the authinfo file.",default='.authinfo') args = parser.parse_args() @@ -84,12 +89,12 @@ badprofile=0 #profile does not exist if access_file==False: - badprofile=1 + badprofile=1 host='default' #profile is empty file -if os.stat(endpointfile).st_size==0: +if os.stat(endpointfile).st_size==0: badprofile=1 host='default' @@ -124,11 +129,11 @@ if profileexists: print('user: '+username) print('profile: '+myprofile) print('host: '+host) - + #quote the password string for posix systems - if (os.name =='posix'): command=clidir+"sas-admin --profile "+myprofile+ " auth login -u "+username+ " -p '"+password+"'" - else: command=clidir+'sas-admin --profile '+myprofile+ ' auth login -u '+username+ ' -p '+password + if (os.name =='posix'): command=clicommand+" --profile "+myprofile+ " auth login -u "+username+ " -p '"+password+"'" + else: command=clicommand+' --profile '+myprofile+ ' auth login -u '+username+ ' -p '+password subprocess.call(command, shell=True) - + else: - print('ERROR: '+fname+' does not exist') + print('ERROR: '+fname+' does not exist') diff --git a/sharedfunctions.py b/sharedfunctions.py index 43c57aa..7d66e6f 100755 --- a/sharedfunctions.py +++ b/sharedfunctions.py @@ -14,16 +14,16 @@ # # Change History # -# 27JAN2018 Comments added -# 29JAN2018 Added simpleresults function -# 31JAN2018 Added the ability to pass contenttype to call_rest_api (now callrestapi) +# 27JAN2018 Comments added +# 29JAN2018 Added simpleresults function +# 31JAN2018 Added the ability to pass contenttype to call_rest_api (now callrestapi) # 31JAN2018 Improved error handling of call_rest_api (now callrestapi) -# 31JAN2018 Deal with situation where json is not returned +# 31JAN2018 Deal with situation where json is not returned # 31JAN2018 Fix a bug when neither json or text is returned -# 02FEB2018 Fix a bug when text is returned -# 12MAR2018 Made simple result print generic -# 20MAR2018 Added some comments -# 20MAR2018 Handle errors when profile and authentication token do not exist +# 02FEB2018 Fix a bug when text is returned +# 12MAR2018 Made simple result print generic +# 20MAR2018 Added some comments +# 20MAR2018 Handle errors when profile and authentication token do not exist # 20May2018 Fixed bug in authentication check # 01jun2018 Deal with empty profile error # 23oct2018 Added print result function @@ -58,29 +58,31 @@ import json import pprint import os import collections +import inspect import re + pp = pprint.PrettyPrinter(indent=4) # validate rest api is not used at this time # not used - + def validaterestapi(baseurl, reqval, reqtype, data={}): - + global result print("The request is a "+reqtype+" request: ",baseurl+reqval) - + json_data=json.dumps(data, ensure_ascii=False) - + print("Data for Request:") print(json_data) - - if (reqtype !="get" or reqtype !="post" or reqtype!="delete" or reqtype!="put"): + + if (reqtype !="get" or reqtype !="post" or reqtype!="delete" or reqtype!="put"): print("NOTE: Invalid method") - + return; - + # callrestapi # this is the main function called many other programs and by the callrestapi program to make the REST calls # change history @@ -88,26 +90,26 @@ def validaterestapi(baseurl, reqval, reqtype, data={}): # 28oct2018 Added stop on error to be able to override stopping processing when an error occurs def callrestapi(reqval, reqtype, acceptType='application/json', contentType='application/json',data={},stoponerror=1): - - + + # get the url from the default profile baseurl=getbaseurl() # get the auth token oaval=getauthtoken(baseurl) - + # build the authorization header head= {'Content-type':contentType,'Accept':acceptType} head.update({"Authorization" : oaval}) - - # maybe this can be removed + + # maybe this can be removed global result # sereliaze the data string for the request to json format json_data=json.dumps(data, ensure_ascii=False) - - # call the rest api using the parameters passed in and the requests python library - + + # call the rest api using the parameters passed in and the requests python library + if reqtype=="get": ret = requests.get(baseurl+reqval,headers=head,data=json_data) elif reqtype=="post": @@ -119,20 +121,20 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app else: result=None print("NOTE: Invalid method") - sys.exit() - - - # response error if status code between these numbers + sys.exit() + + + # response error if status code between these numbers if (400 <= ret.status_code <=599): - - print(ret.text) - result=None - if stoponerror: sys.exit() - - # return the result + + print(ret.text) + result=None + if stoponerror: sys.exit() + + # return the result else: - # is it json - try: + # is it json + try: result=ret.json() except: # is it text @@ -141,11 +143,11 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app except: result=None print("NOTE: No result to print") - - - + + + return result; - + # getfolderid # when a Viya content path is passed in return the id, path and uri # change history @@ -153,11 +155,11 @@ def callrestapi(reqval, reqtype, acceptType='application/json', contentType='app # 08Feb2020 return full json as 4 item in list that is returned def getfolderid(path): - - # build the request parameters + + # build the request parameters reqval="/folders/folders/@item?path="+path reqtype='get' - + callrestapi(reqval,reqtype) if result==None: @@ -169,10 +171,10 @@ def getfolderid(path): targetid=result['id'] targetname=result['name'] targeturi="/folders/folders/"+targetid - + return [targetid,targeturi,targetname,result] - - + + # getbaseurl # from the default profile return the baseurl of the Viya server # change history @@ -180,7 +182,7 @@ def getfolderid(path): # 01jun2018 Deal with empty profile error # 20nov2018 Use the SAS_CLI_PROFILE env variable - + def getbaseurl(): # check that profile file is available and can be read @@ -202,13 +204,13 @@ def getbaseurl(): # get json from profile with open(endpointfile) as json_file: data = json.load(json_file) - + # get the profile environment variable to use it # if it is not set default to the default profile - + cur_profile=os.environ.get("SAS_CLI_PROFILE","Default") #print("URL: ",cur_profile ) - + # check that information is in profile if cur_profile in data: baseurl=data[cur_profile]['sas-endpoint'] @@ -223,57 +225,57 @@ def getbaseurl(): # getauthtoken -# from the stored auth file get the authentication token for the request header +# from the stored auth file get the authentication token for the request header # change history # 01dec2017 initial development # return oaval=None when no authtoken retrieved # 20nov2018 Use the SAS_CLI_PROFILE env variable def getauthtoken(baseurl): - - + + #get authentication information for the header credential_file=os.path.join(os.path.expanduser('~'),'.sas','credentials.json') - + # check that credential file is available and can be read access_file=file_accessible(credential_file,'r') - + if access_file==False: oaval=None print("ERROR: Cannot read authentication credentials at: ", credential_file) print("ERROR: Try refreshing your token with sas-admin auth login") sys.exit() - - with open(credential_file) as json_file: + + with open(credential_file) as json_file: data = json.load(json_file) type(data) - + # the sas-admin profile init creates an empty credential file # check that credential is in file, if it is add it to the header, if not exit - + # get the profile environment variable to use it # if it is not set default to the default profile - + cur_profile=os.environ.get("SAS_CLI_PROFILE","Default") - + #print("LOGON: ", cur_profile ) - + if cur_profile in data: - + oauthToken=data[cur_profile]['access-token'] - + oauthTokenType="bearer" - + oaval=oauthTokenType + ' ' + oauthToken - + head= {'Content-type':'application/json','Accept':'application/json' } head.update({"Authorization" : oaval}) - + # test a connection to rest api if it fails exit r = requests.get(baseurl,headers=head) - + if (400 <= r.status_code <=599): - + oaval=None print(r.text) print("ERROR: cannot connect to "+baseurl+" is your token expired?") @@ -285,69 +287,69 @@ def getauthtoken(baseurl): print("ERROR: access token not in file: ", credential_file) print("ERROR: Try refreshing your token with sas-admin auth login") sys.exit() - + return oaval - + # getinputjson # load the returned json to a python dictionary # change history # 01dec2017 initial development - + def getinputjson(input_file): with open(input_file) as json_file: inputdata = json.load(json_file) - return inputdata - + return inputdata + # simpleresults # take the complex json and create a simple print of the results # change history # 01dec2017 initial development # 20dec2018 simple output now alphibetical order by key - + def simpleresults(resultdata): # print a simplification of the json results - - + + # list of items returned by rest call if 'items' in resultdata: - + total_items=resultdata['count'] - + returned_items=len(resultdata['items']) - + if total_items == 0: print("Note: No items returned.") - - for i in range(0,returned_items): - + + for i in range(0,returned_items): + print ("=====Item ",i,"=======") - + origpairs=resultdata['items'][i] - - test=origpairs.get('description') + + test=origpairs.get('description') if test==None: origpairs['description']='None' - + pairs=collections.OrderedDict(sorted(origpairs.items())) - - + + for key,val in pairs.items(): - - if key != 'links': + + if key != 'links': print(key,end="") print(" = ", val) - + print("Result Summary: Total items available: ",total_items ,"Total items returned: ", returned_items) - + elif 'id' in resultdata: #one item returned by rest call - + for key,val in resultdata.items(): - - if key != 'links': + + if key != 'links': print(key,end="") print(" = ", val) - + else: print("NOTE: No JSON Results Found") @@ -357,123 +359,123 @@ def simpleresults(resultdata): # take the complex json and create a simple table of the results # change history # 01aug2018 initial development -# 19dece2018 print csv in column orderwith only common columns +# 19dece2018 print csv in column orderwith only common columns def csvresults(resultdata,columns=[]): - + if 'items' in resultdata: - + total_items=resultdata['count'] - + returned_items=len(resultdata['items']) - + if total_items == 0: print("Note: No items returned.") - - for i in range(0,returned_items): - - + + for i in range(0,returned_items): + + origpairs=resultdata['items'][i] - + # create an ordered dictionary pairs=collections.OrderedDict() - + # loop thru the column list and insert to a new dictionary in that order # this ensures that colums appear in this order in the csv for keylabel in columns: - + # get the value for the current column curval=origpairs.get(keylabel) - + if curval != None: pairs[keylabel] = curval else: - pairs[keylabel] = 'None' - + pairs[keylabel] = 'None' + numvals=len(columns) z=0 - + # print header row of column names for key,val in pairs.items(): - - z=z+1 - + + z=z+1 + # seperate with comma except last item if z==numvals: sep='' else: sep=',' - + if i==0 and key in columns: print(key,sep,end="") - + print("\n",end="") - + z=0 - + # print rows for key,val in pairs.items(): - + # seperate with comma except last item z=z+1 if z==numvals: sep='' else: sep=',' - + if key != 'links' and key in columns: print('"'+str(val)+'"'+sep, end="") - - + + print("\n",end="") - - + + elif 'id' in resultdata: #one item returned by rest call - + numvals=len(resultdata.items()) z=0 - + for key,val in resultdata.items(): - + # seperate with comma except last item z=z+1 if z==numvals: sep='' else: sep=',' - + if key != 'links': print(key,sep,end="") - + print("\n",end="") - - + + z=0 - + for key,val in resultdata.items(): - + # seperate with comma except last item z=z+1 if z==numvals: sep='' else: sep=',' - + if key != 'links': print('"'+str(val)+'"'+sep,end="") - + print("\n",end="") - + else: print("NOTE: No JSON Results Found") # file_accessible -# Check if a file exists and is accessible. +# Check if a file exists and is accessible. # change history # 01dec2017 initial development - + def file_accessible(filepath, mode): - + try: f = open(filepath, mode) f.close() except IOError as e: return False - + return True # printresult -# prints the results in the style requested +# prints the results in the style requested # change history # 28oct2018 initial development # 22dec2018 add csv columns only relevent for csv output, defaults provided but can be overriden when called @@ -481,10 +483,10 @@ def file_accessible(filepath, mode): def printresult(result,output_style,colsforcsv=["id","name","type","description","creationTimeStamp","modifiedTimeStamp"]): - + # print rest call results if type(result) is dict: - + if output_style=='simple': simpleresults(result) elif output_style=='simplejson': @@ -493,54 +495,54 @@ def printresult(result,output_style,colsforcsv=["id","name","type","description" csvresults(result,columns=colsforcsv) else: print(json.dumps(result,indent=2)) - else: print(result) - + else: print(result) + # getprofileinfo # prints the token expiration, endpoint and current user # change history # 20nov2018 initial development - + def getprofileinfo(myprofile): - - - + + + #get authentication information for the header credential_file=os.path.join(os.path.expanduser('~'),'.sas','credentials.json') - + # check that credential file is available and can be read access_file=file_accessible(credential_file,'r') - + if access_file==False: print("ERROR: Cannot read authentication credentials at: ", credential_file) print("ERROR: Try refreshing your token with sas-admin auth login") sys.exit() - - with open(credential_file) as json_file: + + with open(credential_file) as json_file: data = json.load(json_file) type(data) - + # the sas-admin profile init creates an empty credential file # check that credential is in file, if it is add it to the header, if not exit - + # get the profile environment variable to use it # if it is not set default to the default profile - - + + if myprofile in data: - + expiry=data[myprofile]['expiry'] print("Note your authentication token expires at: "+expiry) - + else: - + print("ERROR: access token not in file: ", credential_file) print("ERROR: Try refreshing your token with sas-admin auth login") sys.exit() - - + + # note the path to the profile is hard-coded right now endpointfile=os.path.join(os.path.expanduser('~'),'.sas','config.json') access_file=file_accessible(endpointfile,'r') @@ -558,30 +560,30 @@ def getprofileinfo(myprofile): # get json from profile with open(endpointfile) as json_file: data = json.load(json_file) - - + + # check that information is in profile if myprofile in data: baseurl=data[myprofile]['sas-endpoint'] print("Endpoint is: "+baseurl) else: print("ERROR: profile "+myprofile+" does not exist. Recreate profile with sas-admin profile init.") - - # build the request parameters + + # build the request parameters reqval="/identities/users/@currentUser" reqtype='get' - + result=callrestapi(reqval,reqtype) - + if result==None: print("NOTE: Not logged in.") - + else: print("Logged on as id: "+ result['id']) print("Logged on as name: "+result['name']) - - - + + + # getpath # when a Viya objectURI is passed in return the path # change history @@ -613,7 +615,7 @@ def getpath(objecturi): return path # getidsanduris -# given a result json structure, return a dictionary with a list of id's and uri's +# given a result json structure, return a dictionary with a list of id's and uri's # change history # 01dec2017 initial development @@ -622,52 +624,52 @@ def getidsanduris(resultdata): resultdict={} resultdict['ids']=[] resultdict['uris']=[] - + # loop the result and add a list of ids and uris to the returned dictionary if 'items' in resultdata: - + total_items=resultdata['count'] returned_items=len(resultdata['items']) if total_items == 0: print("Note: No items returned.") - - for i in range(0,returned_items): - + + for i in range(0,returned_items): + resultdict['ids'].append(resultdata['items'][i]['id']) resultdict['uris'].append(resultdata['items'][i]['uri']) - + return resultdict # simplejsonresults # given a result json structure, remove all the "links" items -# this will return a more readable json output +# this will return a more readable json output # change history # 20feb2020 initial development - + def simplejsonresults(resultdata): - + if 'items' in resultdata: # list of items returned by rest call - - for key in list(resultdata): - if key == 'links': del resultdata[key] + + for key in list(resultdata): + if key == 'links': del resultdata[key] total_items=resultdata['count'] returned_items=len(resultdata['items']) - + if total_items == 0: print("Note: No items returned.") - - for i in range(0,returned_items): - + + for i in range(0,returned_items): + for key in list(resultdata['items'][i]): - if key=='links': + if key=='links': del resultdata['items'][i][key] - + print(json.dumps(resultdata,indent=2)) elif 'id' in resultdata: #one item returned by rest call - del resultdata['links'] + del resultdata['links'] print(json.dumps(resultdata,indent=2)) @@ -684,3 +686,18 @@ def simplejsonresults(resultdata): def get_valid_filename(s): s = str(s).strip().replace(' ', '_') return re.sub(r'(?u)[^-\w.]', '', s) + +# getapplicationproperties +# 20nov2020 initial development + +def getapplicationproperties(): + + # get the path for the script file this is where the properties file will bbe + thepath=os.path.split(inspect.getsourcefile(lambda:0)) + install_dir=thepath[0] + prop_file=os.path.join(install_dir, "application.properties") + + myparams=dict(line.strip().split('=') for line in open(prop_file) if line[0].isalpha()) + return myparams + + diff --git a/showsetup.py b/showsetup.py index 99325e6..c70a985 100755 --- a/showsetup.py +++ b/showsetup.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- # # showsetup.py -# +# # output some system settings to help with debugging issues # # October 2018 @@ -30,7 +30,7 @@ import sys import requests import os -from sharedfunctions import getprofileinfo +from sharedfunctions import getprofileinfo, getapplicationproperties # software versions print("Python Version is: "+str(sys.version_info[0])+'.'+str(sys.version_info[1])) @@ -40,7 +40,7 @@ print("Requests Version is: "+requests.__version__) cur_profile=os.environ.get("SAS_CLI_PROFILE","NOTSET") -if cur_profile=="NOTSET": +if cur_profile=="NOTSET": print("SAS_CLI_PROFILE environment variable not set, using Default profile") cur_profile='Default' else: @@ -49,7 +49,7 @@ else: ssl_file=os.environ.get("SSL_CERT_FILE","NOTSET") -if ssl_file=="NOTSET": +if ssl_file=="NOTSET": print("SSL_CERT_FILE environment variable not set.") else: print("SSL_CERT_FILE environment variable set to profile "+ ssl_file) @@ -57,9 +57,19 @@ else: r_ssl_file=os.environ.get("REQUESTS_CA_BUNDLE","NOTSET") -if r_ssl_file=="NOTSET": +if r_ssl_file=="NOTSET": print("REQUESTS_CA_BUNDLE environment variable not set.") else: print("REQUESTS_CA_BUNDLE environment variable set to profile "+ r_ssl_file) getprofileinfo(cur_profile) + + +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) +print(propertylist) diff --git a/snapshotreports.py b/snapshotreports.py index e1f4503..af9ec93 100755 --- a/snapshotreports.py +++ b/snapshotreports.py @@ -42,13 +42,20 @@ import re import argparse, sys, subprocess, uuid, time, os, glob from datetime import datetime as dt, timedelta as td -from sharedfunctions import getfolderid, callrestapi, getpath, get_valid_filename +from sharedfunctions import getfolderid, callrestapi, getpath, getapplicationproperties, get_valid_filename + # get python version version=int(str(sys.version_info[0])) -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) + # get input parameters parser = argparse.ArgumentParser(description="Export Viya Reports each to its own unique transfer package") @@ -150,7 +157,7 @@ if areyousure.upper() =='Y': json_name=get_valid_filename(path_to_report+resultdata['items'][i]["name"].replace(" ","")+'_'+str(i)) - command=clidir+'sas-admin transfer export -u /reports/reports/'+id+' --name "'+package_name+'"' + command=clicommand+' transfer export -u /reports/reports/'+id+' --name "'+package_name+'"' print(command) subprocess.call(command, shell=True) @@ -160,20 +167,18 @@ if areyousure.upper() =='Y': package_id=package_info['items'][0]['id'] completefile=os.path.join(path,json_name+'.json') - command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id + command=clicommand+' transfer download --file '+completefile+' --id '+package_id print(command) subprocess.call(command, shell=True) #time.sleep(1) if autotranferremove: - print(clidir+'sas-admin transfer delete --id '+package_id+"\n") - remTransferObject = subprocess.Popen(clidir+'sas-admin transfer delete --id '+package_id, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) + print(clicommand+' transfer delete --id '+package_id+"\n") + remTransferObject = subprocess.Popen(clicommand+' transfer delete --id '+package_id, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True) remTransferObjectOutput = remTransferObject.communicate(b'Y\n') remTransferObject.wait() - print("NOTE: "+str(reports_exported)+" report(s) exported to json files in "+path) print("NOTE: "+str(total_items)+" total reports found, "+str(reports_exported)+" reports exported to json files in "+path) - else: print("NOTE: Operation cancelled") diff --git a/testfolderaccess.py b/testfolderaccess.py index a24d70e..3a2bbee 100755 --- a/testfolderaccess.py +++ b/testfolderaccess.py @@ -24,9 +24,6 @@ # limitations under the License. # -# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION -clidir='/opt/sas/viya/home/bin/' -debug=False # Import Python modules @@ -34,8 +31,23 @@ import argparse import subprocess import json import sys +import os + +from sharedfunctions import getfolderid,callrestapi,getapplicationproperties + + +# get python version +version=int(str(sys.version_info[0])) -from sharedfunctions import getfolderid,callrestapi +# get cli location from properties +propertylist=getapplicationproperties() + +clidir=propertylist["sascli.location"] +cliexe=propertylist["sascli.executable"] + +clicommand=os.path.join(clidir,cliexe) + +debug=False # Define exception handler so that we only output trace info from errors when in debug mode def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook): diff --git a/updatepreferences.py b/updatepreferences.py index b60ac42..e64e176 100755 --- a/updatepreferences.py +++ b/updatepreferences.py @@ -8,7 +8,7 @@ # # Change History # -# 30OCT2018 first version +# 30OCT2018 first version # # Copyright © 2018, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. # @@ -36,18 +36,18 @@ #################################################################### #### POSSIBLE VALUES #### #################################################################### -#### sas.welcome.suppress = true/false #### -#### sas.drive.show.pinned = true/false #### -#### VA.geo.drivedistance.unit = kilometers/miles #### -#### OpenUI.Theme.Default = sas_corporate/sas_inspire/sas_hcb #### +#### sas.welcome.suppress = true/false #### +#### sas.drive.show.pinned = true/false #### +#### VA.geo.drivedistance.unit = kilometers/miles #### +#### OpenUI.Theme.Default = sas_corporate/sas_inspire/sas_hcb #### #################################################################### import argparse from sharedfunctions import callrestapi parser = argparse.ArgumentParser(description="Update user preferences for a user or a group of users") -parser.add_argument("-t", "--target", help="Type the target of the update: user or group", required=True, choices=['user', 'group']) -parser.add_argument("-tn", "--targetname", help="ID of the user or group to which the update applies.", required=True) +parser.add_argument("-t", "--target", help="Type the target of the update: user or group", required=True, choices=['user', 'group','all']) +parser.add_argument("-tn", "--targetname", help="ID of the user or group to which the update applies.") parser.add_argument("-pi", "--preferenceid", help="ID of the preference to be updated", required=True) parser.add_argument("-pv", "--preferencevalue", help="Value to be set for the preference", required=True) @@ -59,41 +59,62 @@ preferenceValue = args.preferencevalue json= {"application": "SAS Visual Analytics", "version": 1,"id": preferenceID ,"value": preferenceValue} -# Function to update preference of a specific user -if target == 'user' : - + +# apply for all users in a deployment +if target=='all' : + + reqtype='get' + reqval='/identities/users/?limit=10000' + resultdata=callrestapi(reqval,reqtype) + + reqtype="put" + + if 'items' in resultdata: + + returned_items=len(resultdata['items']) + for i in range(0,returned_items): + + id=resultdata['items'][i]['id'] + type=resultdata['items'][i]['type'] + + if type=="user": + reqval="/preferences/preferences/"+ id +"/" + preferenceID + result=callrestapi(reqval, reqtype,data=json,stoponerror=0) + print("Updating Preference "+reqval+" = "+preferenceValue) + +elif target == 'user' : + userID=targetName - + reqtype='get' reqval="/identities/users/"+userID - + userexist=callrestapi(reqval,reqtype) - + reqtype="put" reqval="/preferences/preferences/"+ userID +"/" + preferenceID result=callrestapi(reqval,reqtype,data=json) print("Updating Preference "+reqval+" = "+preferenceValue) - - + + else: # Execute actual code to update the preference for a user or a group - + reqtype='get' - reqval='/identities/groups/'+ targetName +'/members?limit=1000' - resultdata=callrestapi(reqval,reqtype) - + reqval='/identities/groups/'+ targetName +'/members?limit=1000&depth=-1' + resultdata=callrestapi(reqval,reqtype,contentType="application/vnd.sas.identity.group.member.flat") + reqtype="put" - + if 'items' in resultdata: - + returned_items=len(resultdata['items']) - for i in range(0,returned_items): - - id=resultdata['items'][i]['id'] + for i in range(0,returned_items): + + id=resultdata['items'][i]['id'] type=resultdata['items'][i]['type'] - + if type=="user": reqval="/preferences/preferences/"+ id +"/" + preferenceID result=callrestapi(reqval, reqtype,data=json,stoponerror=0) - print(result) print("Updating Preference "+reqval+" = "+preferenceValue) - else: print("Cannot set preferences for a group "+id ) \ No newline at end of file + else: print("Cannot set preferences for a group "+id )