Viya4 v1 (#64)

* add properties file

* add get properties function

* add get of properties to listreports

* update listreports

* fix

* add insepct

* test with foldertree

* update properties

* test export folder tree

* exportfoldertree

* make cli location and command dynamic

* add imports

* fix

* add os

* add all option

* updates

* update import packages

* update import packages

* a few more sas-viya changes

* add function

* showsetup

* make targetname optional

* update

* update

* Update sharedfunctions.py

added  get_valid_filename

* Update snapshotreports.py

* Update updatepreferences.py

upped the limit on all users

* Update INSTALL.md

* Update INSTALL.md

INSTALL.md

@@ -10,6 +10,24 @@ The tools should be installed on the same machine that hosts the Viya command-li
 
 *git clone https://github.com/sassoftware/pyviyatools.git*
 
+NOTE: to use the tools with Viya 4 clone the viya4_v1 branch
+
+*git clone https://github.com/sassoftware/pyviyatools.git -b viya4_v1*
+
+**Configure** 
+
+The application.properties file contains the default location of the sas-admin or viya cli. Edit this file to match the location of the cli and the cli name in your environment.
+
+The default values for Viya 3.x are:
+
+sascli.location=/opt/sas/viya/home/bin/
+sascli.executable=sas-admin
+
+The default values for Viya 4.x are:
+
+sascli.location=/opt/sas/viya/home/bin/
+sascli.executable=sas-viya
+
 **Authenticate**
 
 The pyviya tools use the sas-admin auth CLI to authenticate to Viya. To use the tool you must create a profile and authenticate. This process is documented in the SAS Viya Administration guide here.

application.properties

@@ -0,0 +1,2 @@
+sascli.location=/opt/sas/viya/home/bin/
+sascli.executable=sas-viya

applyfolderauthorization.py

@@ -38,11 +38,16 @@ import csv
 import os
 import json
 import subprocess
-from sharedfunctions import callrestapi, getfolderid, file_accessible, printresult
+import sys
+from sharedfunctions import callrestapi, getfolderid, file_accessible, printresult,getapplicationproperties
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get cli location from properties
-clidir='/opt/sas/viya/home/bin/'
+propertylist=getapplicationproperties()
-#clidir='c:\\admincli\\'
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
 
 
 # setup command-line arguements
@@ -112,6 +117,6 @@ with open("bulk_rules_list.json", "w") as text_file:
     text_file.write(bulk_rules_list_string+'\n')
 
 # Execute sas-admin CLI to apply rules from JSON schema
-command=clidir+'sas-admin authorization create-rules --file bulk_rules_list.json'
+command=clicommand+' authorization create-rules --file bulk_rules_list.json'
 print("Executing command: "+command)
 subprocess.call(command, shell=True)

createbinarybackup.py

@@ -23,8 +23,19 @@
 #  limitations under the License.
 #
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get python version
-clidir='/opt/sas/viya/home/bin/'
+version=int(str(sys.version_info[0]))
+
+# get cli location from properties
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+
+
+
 debug=False
 defaultBackupScheduleName="DEFAULT_BACKUP_SCHEDULE"
 newScheduleName="BINARY_BACKUP_SCHEDULE"

explainaccess.py

@@ -54,8 +54,25 @@
 #  limitations under the License.
 #
 
+# Import Python modules
+
+import argparse
+import subprocess
+import json
+import sys
+from sharedfunctions import getfolderid,callrestapi,getapplicationproperties
+
+# get python version
+version=int(str(sys.version_info[0]))
+
+# get cli location from properties
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
 
-clidir='/opt/sas/viya/home/bin/'
 debug=False
 direct_only=False
 valid_permissions=['read','update','delete','secure','add','remove','create']
@@ -64,15 +81,6 @@ default_permissions=['read','update','delete','secure','add','remove']
 direct_permission_suffix='*'
 
 
-# Import Python modules
-
-import argparse
-import subprocess
-import json
-import sys
-
-from sharedfunctions import getfolderid,callrestapi
-
 # Define exception handler so that we only output trace info from errors when in debug mode
 def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook):
     if debug:

exportfoldertree.py

@@ -30,13 +30,18 @@
 # Import Python modules
 import argparse, sys, subprocess, uuid, time, os, glob
 
-from sharedfunctions import getfolderid, callrestapi
+from sharedfunctions import getfolderid, callrestapi,getapplicationproperties
 
 # get python version
 version=int(str(sys.version_info[0]))
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get cli location from properties
-clidir='/opt/sas/viya/home/bin/'
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
 
 # get input parameters
 parser = argparse.ArgumentParser(description="Export the complete Viya folder tree")
@@ -95,7 +100,7 @@ if areyousure.upper() =='Y':
 				package_name=str(uuid.uuid1())
 				json_name=resultdata['items'][i]["name"].replace(" ","")+'_'+str(i)
 
-				command=clidir+'sas-admin transfer export -u /folders/folders/'+id+' --name "'+package_name+'"'
+				command=clicommand+' transfer export -u /folders/folders/'+id+' --name "'+package_name+'"'
 				print(command)
 				subprocess.call(command, shell=True)
 
@@ -105,7 +110,7 @@ if areyousure.upper() =='Y':
 				package_id=package_info['items'][0]['id']
 
 				completefile=os.path.join(path,json_name+'.json')
-				command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id    
+				command=clicommand+' transfer download --file '+completefile+' --id '+package_id
 				print(command)
 				subprocess.call(command, shell=True)
 

getpath.py

@@ -31,7 +31,17 @@
 #
 
 
-clidir='/opt/sas/viya/home/bin/'
+# get python version
+version=int(str(sys.version_info[0]))
+
+# get cli location from properties
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+
 debug=False
 
 # Import Python modules

importpackages.py

@@ -23,10 +23,15 @@
 #
 # Import Python modules
 import argparse, sys, subprocess, os, json
-from sharedfunctions import callrestapi
+from sharedfunctions import callrestapi, getapplicationproperties
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get cli location from properties
-clidir='/opt/sas/viya/home/bin/'
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
 
 # get input parameters
 parser = argparse.ArgumentParser(description="Import JSON files from directory. All json files in directory will be imported.")
@@ -62,7 +67,7 @@ if areyousure.upper() =='Y':
 			if filename.lower().endswith('.json'):
 
 				#upload the json package
-				command=clidir+'sas-admin transfer upload --file '+os.path.join(basedir,filename)+'> /tmp/packageid.json'
+				command=clicommand+' transfer upload --file '+os.path.join(basedir,filename)+'> /tmp/packageid.json'
 				print(command)
 				subprocess.call(command, shell=True)
 
@@ -74,7 +79,7 @@ if areyousure.upper() =='Y':
 
 				# get the packageid and import the package
 				packageid=package_data["id"]
-				command=clidir+'sas-admin --output text -q transfer import --id '+packageid
+				command=clicommand+' --output text -q transfer import --id '+packageid
 				print(command)
 
 				subprocess.call(command, shell=True)

listreports.py

@@ -27,14 +27,11 @@
 # Import Python modules
 import argparse, sys, subprocess, uuid, time, os, glob
 from datetime import datetime as dt, timedelta as td
-from sharedfunctions import getfolderid, callrestapi, getpath, printresult
+from sharedfunctions import getfolderid, callrestapi, getpath, printresult, getapplicationproperties
 
 # get python version
 version=int(str(sys.version_info[0]))
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
-clidir='/opt/sas/viya/home/bin/'
-
 # get input parameters
 parser = argparse.ArgumentParser(description="List Viya Reports and their folder path.")
 parser.add_argument("-n","--name", help="Name contains?",default=None)

loginviauthinfo.py

@@ -47,7 +47,7 @@ import os
 import argparse
 import json
 
-from sharedfunctions import file_accessible
+from sharedfunctions import file_accessible,getapplicationproperties
 
 try:
     # Python 3
@@ -57,9 +57,14 @@ except ImportError:
     from urlparse import urlparse
 
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get cli location from properties
-clidir='/opt/sas/viya/home/bin/'
+propertylist=getapplicationproperties()
-#clidir='c:\\admincli\\'
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+
 
 debug=0
 profileexists=0
@@ -126,8 +131,8 @@ if profileexists:
           print('host: '+host)
 
        #quote the password string for posix systems
-       if (os.name =='posix'): command=clidir+"sas-admin  --profile "+myprofile+ " auth login -u "+username+ " -p '"+password+"'"
+       if (os.name =='posix'): command=clicommand+"  --profile "+myprofile+ " auth login -u "+username+ " -p '"+password+"'"
-       else: command=clidir+'sas-admin --profile '+myprofile+ ' auth login -u '+username+ ' -p '+password 
+       else: command=clicommand+' --profile '+myprofile+ ' auth login -u '+username+ ' -p '+password
        subprocess.call(command, shell=True)
 
     else:

sharedfunctions.py

@@ -58,8 +58,10 @@ import json
 import pprint
 import os
 import collections
+import inspect
 import re
 
+
 pp = pprint.PrettyPrinter(indent=4)
 
 # validate rest api is not used at this time
@@ -684,3 +686,18 @@ def simplejsonresults(resultdata):
 def get_valid_filename(s):
 	s = str(s).strip().replace(' ', '_')
 	return re.sub(r'(?u)[^-\w.]', '', s)
+
+# getapplicationproperties
+#   20nov2020 initial development
+
+def getapplicationproperties():
+
+    # get the path for the script file this is where the properties file will bbe
+    thepath=os.path.split(inspect.getsourcefile(lambda:0))
+    install_dir=thepath[0]
+    prop_file=os.path.join(install_dir, "application.properties")
+
+    myparams=dict(line.strip().split('=') for line in open(prop_file) if line[0].isalpha())
+    return myparams
+
+

showsetup.py

@@ -30,7 +30,7 @@ import sys
 import requests
 import os
 
-from sharedfunctions import getprofileinfo
+from sharedfunctions import getprofileinfo, getapplicationproperties
 
 # software versions
 print("Python Version is: "+str(sys.version_info[0])+'.'+str(sys.version_info[1]))
@@ -63,3 +63,13 @@ else:
     print("REQUESTS_CA_BUNDLE environment variable set to profile "+ r_ssl_file)
 
 getprofileinfo(cur_profile)
+
+
+# get cli location from properties
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+print(propertylist)

snapshotreports.py

@@ -42,13 +42,20 @@
 import re
 import argparse, sys, subprocess, uuid, time, os, glob
 from datetime import datetime as dt, timedelta as td
-from sharedfunctions import getfolderid, callrestapi, getpath, get_valid_filename
+from sharedfunctions import getfolderid, callrestapi, getpath, getapplicationproperties, get_valid_filename
+
 
 # get python version
 version=int(str(sys.version_info[0]))
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
+# get cli location from properties
-clidir='/opt/sas/viya/home/bin/'
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+
 
 # get input parameters
 parser = argparse.ArgumentParser(description="Export Viya Reports each to its own unique transfer package")
@@ -150,7 +157,7 @@ if areyousure.upper() =='Y':
 
 					json_name=get_valid_filename(path_to_report+resultdata['items'][i]["name"].replace(" ","")+'_'+str(i))
 
-					command=clidir+'sas-admin transfer export -u /reports/reports/'+id+' --name "'+package_name+'"'
+					command=clicommand+' transfer export -u /reports/reports/'+id+' --name "'+package_name+'"'
 					print(command)
 					subprocess.call(command, shell=True)
 
@@ -160,20 +167,18 @@ if areyousure.upper() =='Y':
 					package_id=package_info['items'][0]['id']
 
 					completefile=os.path.join(path,json_name+'.json')
-					command=clidir+'sas-admin transfer download --file '+completefile+' --id '+package_id
+					command=clicommand+' transfer download --file '+completefile+' --id '+package_id
 					print(command)
 					subprocess.call(command, shell=True)
 					#time.sleep(1)
 					if autotranferremove:
-						print(clidir+'sas-admin transfer delete --id '+package_id+"\n")
+						print(clicommand+' transfer delete --id '+package_id+"\n")
-						remTransferObject = subprocess.Popen(clidir+'sas-admin transfer delete --id '+package_id, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
+						remTransferObject = subprocess.Popen(clicommand+' transfer delete --id '+package_id, stdin=subprocess.PIPE, stdout=subprocess.PIPE, shell=True)
 						remTransferObjectOutput = remTransferObject.communicate(b'Y\n')
 						remTransferObject.wait()
 
 
-					print("NOTE: "+str(reports_exported)+" report(s) exported to json files in "+path)
 			print("NOTE: "+str(total_items)+" total reports found, "+str(reports_exported)+" reports exported to json files in "+path)
 
-
 else:
 	 print("NOTE: Operation cancelled")

testfolderaccess.py

@@ -24,9 +24,6 @@
 #  limitations under the License.
 #
 
-# CHANGE THIS VARIABLE IF YOUR CLI IS IN A DIFFERENT LOCATION
-clidir='/opt/sas/viya/home/bin/'
-debug=False
 
 # Import Python modules
 
@@ -34,8 +31,23 @@ import argparse
 import subprocess
 import json
 import sys
+import os
+
+from sharedfunctions import getfolderid,callrestapi,getapplicationproperties
+
+
+# get python version
+version=int(str(sys.version_info[0]))
 
-from sharedfunctions import getfolderid,callrestapi
+# get cli location from properties
+propertylist=getapplicationproperties()
+
+clidir=propertylist["sascli.location"]
+cliexe=propertylist["sascli.executable"]
+
+clicommand=os.path.join(clidir,cliexe)
+
+debug=False
 
 # Define exception handler so that we only output trace info from errors when in debug mode
 def exception_handler(exception_type, exception, traceback, debug_hook=sys.excepthook):

updatepreferences.py

@@ -46,8 +46,8 @@ import argparse
 from sharedfunctions import callrestapi
 
 parser = argparse.ArgumentParser(description="Update user preferences for a user or a group of users")
-parser.add_argument("-t", "--target", help="Type the target of the update: user or group", required=True, choices=['user', 'group'])
+parser.add_argument("-t", "--target", help="Type the target of the update: user or group", required=True, choices=['user', 'group','all'])
-parser.add_argument("-tn", "--targetname", help="ID of the user or group to which the update applies.", required=True)
+parser.add_argument("-tn", "--targetname", help="ID of the user or group to which the update applies.")
 parser.add_argument("-pi", "--preferenceid", help="ID of the preference to be updated", required=True)
 parser.add_argument("-pv", "--preferencevalue", help="Value to be set for the preference", required=True)
 
@@ -59,8 +59,30 @@ preferenceValue = args.preferencevalue
 
 json= {"application": "SAS Visual Analytics", "version": 1,"id": preferenceID ,"value": preferenceValue}
 
-# Function to update preference of a specific user
+
-if target == 'user' :
+# apply for all users in a deployment
+if target=='all' :
+
+    reqtype='get'
+    reqval='/identities/users/?limit=10000'
+    resultdata=callrestapi(reqval,reqtype)
+
+    reqtype="put"
+
+    if 'items' in resultdata:
+
+        returned_items=len(resultdata['items'])
+        for i in range(0,returned_items):
+
+           id=resultdata['items'][i]['id']
+           type=resultdata['items'][i]['type']
+
+           if type=="user":
+               reqval="/preferences/preferences/"+ id +"/" + preferenceID
+               result=callrestapi(reqval, reqtype,data=json,stoponerror=0)
+               print("Updating Preference "+reqval+" = "+preferenceValue)
+
+elif target == 'user' :
 
     userID=targetName
 
@@ -78,8 +100,8 @@ if target == 'user' :
 else: # Execute actual code to update the preference for a user or a group
 
     reqtype='get'
-    reqval='/identities/groups/'+ targetName +'/members?limit=1000'
+    reqval='/identities/groups/'+ targetName +'/members?limit=1000&depth=-1'
-    resultdata=callrestapi(reqval,reqtype)
+    resultdata=callrestapi(reqval,reqtype,contentType="application/vnd.sas.identity.group.member.flat")
 
     reqtype="put"
 
@@ -94,6 +116,5 @@ else: # Execute actual code to update the preference for a user or a group
            if type=="user":
                reqval="/preferences/preferences/"+ id +"/" + preferenceID
                result=callrestapi(reqval, reqtype,data=json,stoponerror=0)
-               print(result)
                print("Updating Preference "+reqval+" = "+preferenceValue)
            else: print("Cannot set preferences for a group "+id )