From 8e54e9165634785aacff098961e2ea0dc20c72fc Mon Sep 17 00:00:00 2001 From: Gerry Nelson <45608078+gerrynelson63@users.noreply.github.com> Date: Mon, 10 Oct 2022 12:21:30 -0400 Subject: [PATCH] Dev/fileservice (#117) * list files add cols and sort * bug * day subset * add listcontent.py * parentfolderudi * add folder path * print root folders * remove folders from output + verbose * path to item verbose Co-authored-by: cloud-user --- listcontent.py | 142 +++++++++++++++++++++++++++++++++++++++++++++ listfiles.py | 36 +++++++----- sharedfunctions.py | 21 +++---- 3 files changed, 176 insertions(+), 23 deletions(-) create mode 100755 listcontent.py diff --git a/listcontent.py b/listcontent.py new file mode 100755 index 0000000..22540d7 --- /dev/null +++ b/listcontent.py @@ -0,0 +1,142 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +# +# listcontent.py +# february 2018 +# +# Pass in a folder path and list content +# Change History + +# +# Copyright © 2018, SAS Institute Inc., Cary, NC, USA. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the License); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Import Python modules + +import argparse, sys + +from sharedfunctions import getfolderid, callrestapi, printresult, getfolderid, getidsanduris, getpath, json +from datetime import datetime as dt, timedelta as td + +# get python version +version=int(str(sys.version_info[0])) + +# get input parameters +parser = argparse.ArgumentParser(description="List folder and its sub-folders and content.") +parser.add_argument("-f","--folderpath", help="Enter the path to the viya folder to start the listing.",required='True') +parser.add_argument("-v","--verbosecsv", help="Verbose CSV(only used with -o=csv) ", action='store_true' ) +parser.add_argument("-o","--output", help="Output Style", choices=['csv','json','simple','simplejson'],default='json') +parser.add_argument("--debug", action='store_true', help="Debug") + +args = parser.parse_args() +debug=args.debug +path_to_folder=args.folderpath +verbosecsv=args.verbosecsv +output_style=args.output + +delimiter = ',' + +if verbosecsv: cols=cols=["id","pathtoitem","name","type","contentType","description","typeDefName","documentType","contentDisposition","fileStatus","searchable","size","createdBy","creationTimeStamp","modifiedBy","modifiedTimeStamp","expirationTimeStamp","encoding","parentUri"] +else: cols=["id","pathtoitem","name","contentType","createdBy","creationTimeStamp","modifiedBy","modifiedTimeStamp","uri"] + +def getfoldercontent(path_to_folder): + + # call getfolderid to get the folder id + targets=getfolderid(path_to_folder) + + #if debug: print(targets) + + # if the folder is found + if targets[0] is not None: + + uri=targets[1] + + #get folder content, recursive call returns all children + reqval=uri+"/members?recursive=true&limit=100000" + reqtype='get' + + if debug: print(reqval) + folders_result_json=callrestapi(reqval,reqtype) + + # add a loop of folder result that adds the parent folder path + # add the path back into the json for printing + total_items=folders_result_json['count'] + itemlist=folders_result_json['items'] + returned_items=len(itemlist) + + for i in range(0,returned_items): + + contenttype=itemlist[i]["contentType"] + itemuri=itemlist[i]["uri"] + name=itemlist[i]["name"] + + parentFolderUri=itemlist[i]["parentFolderUri"] + path_to_item=getpath(itemuri) + + itemlist[i]["pathtoitem"]=path_to_item + itemlist[i]["pathanditemname"]=path_to_item+name + #print(path_to_item,name,contenttype) + + newitems = [ ] + + # remove folders + for i in range(0,returned_items): + + if folders_result_json['items'][i]["contentType"]!="folder": + newitems.append(folders_result_json['items'][i]) + + folders_result_json["count"]=len(newitems) + folders_result_json["items"]=newitems + + return folders_result_json + +# root folder, loop sub-folders and print +if path_to_folder=="/": + + # get a list of root folders + + # get the json result for each one + + # append them together for printing + + reqval='/folders/rootFolders' + reqtype='get' + rootfolderresult=callrestapi(reqval,reqtype) + rootfolders=rootfolderresult["items"] + + z=0 + + for folder in rootfolders: + + foldername="/"+folder["name"] + result_json=getfoldercontent(foldername) + + content_count=result_json['count'] + + # print if there is content in the folder + if content_count: + + # printer header only for the first group + if z==0: printresult(result_json,output_style,cols) + else: printresult(result_json,output_style,cols,header=0) + z=z+1 + +else: + + # print folder content + result_json=getfoldercontent(path_to_folder) + printresult(result_json,output_style,cols) + + diff --git a/listfiles.py b/listfiles.py index d54c698..b775700 100755 --- a/listfiles.py +++ b/listfiles.py @@ -42,27 +42,32 @@ from datetime import datetime as dt, timedelta as td parser = argparse.ArgumentParser() -parser = argparse.ArgumentParser(description="Query and list files stored in the infrastructure data server.") +parser = argparse.ArgumentParser(description="Query and list files stored in the Viya Infrastructure Data sSrver.") parser.add_argument("-n","--name", help="Name contains",default=None) parser.add_argument("-c","--type", help="Content Type in.",default=None) parser.add_argument("-p","--parent", help="ParentURI starts with.",default=None) parser.add_argument("-pf","--parentfolder", help="Parent Folder Name.",default=None) -parser.add_argument("-d","--days", help="List files older than this number of days",default='-1') +parser.add_argument("-d","--days", help="List files older or younger than this number of days",default='-1') +parser.add_argument("-do","--olderoryounger", help="For the date subsetting specify older or younger",choices=['older','younger'],default='older') parser.add_argument("-m","--modifiedby", help="Last modified id equals",default=None) -parser.add_argument("-s","--sortby", help="Sort the output descending by this field",default='modifiedTimeStamp') +parser.add_argument("-s","--sortby", help="Sort the output by this field",default='modifiedTimeStamp') +parser.add_argument("-so","--sortorder", help="Sort order",choices=['ascending','descending'],default='descending') +parser.add_argument("-v","--verbosecsv", help="Verbose CSV(only used with -o=csv) ", action='store_false' ) parser.add_argument("-o","--output", help="Output Style", choices=['csv','json','simple','simplejson'],default='json') parser.add_argument("--debug", action='store_true', help="Debug") args = parser.parse_args() output_style=args.output -daysolder=args.days +days=args.days modby=args.modifiedby sortby=args.sortby nameval=args.name puri=args.parent pfolder=args.parentfolder debug=args.debug - +verbosecsv=args.verbosecsv +sortorder=args.sortorder +olderoryounger=args.olderoryounger files_result_json=None @@ -73,9 +78,12 @@ if puri !=None and pfolder !=None: sys.exit() # calculate time period for files -now=dt.today()-td(days=int(daysolder)) +now=dt.today()-td(days=int(days)) subset_date=now.strftime("%Y-%m-%dT%H:%M:%S") -datefilter="le(creationTimeStamp,"+subset_date+")" + +if olderoryounger=='older': + datefilter="le(creationTimeStamp,"+subset_date+")" +else: datefilter="ge(creationTimeStamp,"+subset_date+")" # create a list for filter conditions filtercond=[] @@ -94,7 +102,7 @@ delimiter = ',' if puri!=None: filtercond.append("contains(parentUri,'"+puri+"')") completefilter = 'and('+delimiter.join(filtercond)+')' - reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000" + reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000" # process items in folders elif pfolder!=None: @@ -102,7 +110,7 @@ elif pfolder!=None: folderid=getfolderid(pfolder)[0] # add the start and end and comma delimit the filter completefilter = 'and('+delimiter.join(filtercond)+')' - reqval="/folders/folders/"+folderid+"/members?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000" + reqval="/folders/folders/"+folderid+"/members?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000" files_in_folder=callrestapi(reqval,reqtype) @@ -126,18 +134,20 @@ elif pfolder!=None: filtercond.append("in(id,"+inclause+")") completefilter = 'and('+delimiter.join(filtercond)+')' - reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000" + reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"g&limit=10000" else: completefilter = 'and('+delimiter.join(filtercond)+')' - reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000" + reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000" if debug: print(reqval) files_result_json=callrestapi(reqval,reqtype) - -cols=['id','name','contentType','documentType','createdBy','modifiedTimeStamp','size','parentUri'] +if verbosecsv: + cols=['id','name','contentType','documentType','createdBy','modifiedTimeStamp','size','parentUri'] +else: + cols=['id','name','contentType','description','typeDefName','documentType','contentDisposition','fileStatus','searchable','size','creationTimeStamp','createdBy','modifiedBy','modifiedTimeStamp','expirationTimeStamp','encoding','parentUri'] # print result if files_result_json == None: diff --git a/sharedfunctions.py b/sharedfunctions.py index 599e34b..4db7e1d 100755 --- a/sharedfunctions.py +++ b/sharedfunctions.py @@ -387,7 +387,7 @@ def simpleresults(resultdata): # 01aug2018 initial development # 19dece2018 print csv in column orderwith only common columns -def csvresults(resultdata,columns=[]): +def csvresults(resultdata,columns=[],header=1): if 'items' in resultdata: @@ -421,16 +421,17 @@ def csvresults(resultdata,columns=[]): numvals=len(columns) z=0 - # print header row of column names - for key,val in pairs.items(): + if header: + # print header row of column names + for key,val in pairs.items(): - z=z+1 + z=z+1 - # seperate with comma except last item - if z==numvals: sep='' - else: sep=',' + # seperate with comma except last item + if z==numvals: sep='' + else: sep=',' - if i==0 and key in columns: print(key,sep,end="") + if i==0 and key in columns: print(key,sep,end="") print("\n",end="") @@ -519,7 +520,7 @@ def file_accessible(filepath, mode): # 22dec2018 add csv columns only relevent for csv output, defaults provided but can be overriden when called # 20feb2020 add simplejson output style -def printresult(result,output_style,colsforcsv=["id","name","type","description","creationTimeStamp","modifiedTimeStamp"]): +def printresult(result,output_style,colsforcsv=["id","name","type","description","creationTimeStamp","modifiedTimeStamp"],header=1): # print rest call results @@ -530,7 +531,7 @@ def printresult(result,output_style,colsforcsv=["id","name","type","description" elif output_style=='simplejson': simplejsonresults(result) elif output_style=='csv': - csvresults(result,columns=colsforcsv) + csvresults(result,columns=colsforcsv,header=header) else: print(json.dumps(result,indent=2)) else: print(result)