Dev/fileservice (#117)

* list files add cols and sort

* bug

* day subset

* add listcontent.py

* parentfolderudi

* add folder path

* print root folders

* remove folders from output + verbose

* path to item verbose

Co-authored-by: cloud-user <cloud-user@race.sas.com>

listcontent.py

@@ -0,0 +1,142 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+#
+# listcontent.py
+# february 2018
+#
+# Pass in a folder path and list content
+# Change History
+
+#
+# Copyright © 2018, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
+#
+#  Licensed under the Apache License, Version 2.0 (the License);
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+# Import Python modules
+
+import argparse, sys
+
+from sharedfunctions import getfolderid, callrestapi, printresult, getfolderid, getidsanduris, getpath, json
+from datetime import datetime as dt, timedelta as td
+
+# get python version  
+version=int(str(sys.version_info[0]))
+
+# get input parameters	
+parser = argparse.ArgumentParser(description="List folder and its sub-folders and content.")
+parser.add_argument("-f","--folderpath", help="Enter the path to the viya folder to start the listing.",required='True')
+parser.add_argument("-v","--verbosecsv", help="Verbose CSV(only used with -o=csv) ", action='store_true' )
+parser.add_argument("-o","--output", help="Output Style", choices=['csv','json','simple','simplejson'],default='json')
+parser.add_argument("--debug", action='store_true', help="Debug")
+
+args = parser.parse_args()
+debug=args.debug
+path_to_folder=args.folderpath
+verbosecsv=args.verbosecsv
+output_style=args.output
+
+delimiter = ','
+
+if verbosecsv: cols=cols=["id","pathtoitem","name","type","contentType","description","typeDefName","documentType","contentDisposition","fileStatus","searchable","size","createdBy","creationTimeStamp","modifiedBy","modifiedTimeStamp","expirationTimeStamp","encoding","parentUri"]
+else: cols=["id","pathtoitem","name","contentType","createdBy","creationTimeStamp","modifiedBy","modifiedTimeStamp","uri"]
+
+def getfoldercontent(path_to_folder):
+
+    # call getfolderid to get the folder id
+    targets=getfolderid(path_to_folder)
+
+    #if debug: print(targets)
+
+    # if the folder is found
+    if targets[0] is not None:
+        
+        uri=targets[1]
+        
+        #get folder content, recursive call returns all children
+        reqval=uri+"/members?recursive=true&limit=100000"
+        reqtype='get'
+
+        if debug: print(reqval)
+        folders_result_json=callrestapi(reqval,reqtype)
+        
+        # add a loop of folder result that adds the parent folder path
+        # add the path back into the json for printing
+        total_items=folders_result_json['count']
+        itemlist=folders_result_json['items']
+        returned_items=len(itemlist)
+        
+        for i in range(0,returned_items):
+
+            contenttype=itemlist[i]["contentType"]
+            itemuri=itemlist[i]["uri"]
+            name=itemlist[i]["name"]
+                              
+            parentFolderUri=itemlist[i]["parentFolderUri"]   
+            path_to_item=getpath(itemuri)
+
+            itemlist[i]["pathtoitem"]=path_to_item
+            itemlist[i]["pathanditemname"]=path_to_item+name
+            #print(path_to_item,name,contenttype) 
+
+        newitems = [ ]
+
+        # remove folders
+        for i in range(0,returned_items):
+
+            if folders_result_json['items'][i]["contentType"]!="folder":
+                newitems.append(folders_result_json['items'][i])
+            
+        folders_result_json["count"]=len(newitems)
+        folders_result_json["items"]=newitems
+     
+    return folders_result_json
+
+# root folder, loop sub-folders and print
+if path_to_folder=="/":
+       
+    # get a list of root folders
+
+    # get the json result for each one
+
+    # append them together for printing
+
+    reqval='/folders/rootFolders'
+    reqtype='get'
+    rootfolderresult=callrestapi(reqval,reqtype)
+    rootfolders=rootfolderresult["items"]
+   
+    z=0
+
+    for folder in rootfolders:
+        
+        foldername="/"+folder["name"]
+        result_json=getfoldercontent(foldername)
+
+        content_count=result_json['count']
+        
+        # print if there is content in the folder
+        if content_count: 
+
+            # printer header only for the first group    
+            if z==0: printresult(result_json,output_style,cols)
+            else: printresult(result_json,output_style,cols,header=0)
+            z=z+1
+            
+else:
+    
+    # print folder content
+    result_json=getfoldercontent(path_to_folder)
+    printresult(result_json,output_style,cols)
+            
+

listfiles.py

@@ -42,27 +42,32 @@ from datetime import datetime as dt, timedelta as td
 
 parser = argparse.ArgumentParser()
 
-parser = argparse.ArgumentParser(description="Query and list files stored in the infrastructure data server.")
+parser = argparse.ArgumentParser(description="Query and list files stored in the Viya Infrastructure Data sSrver.")
 parser.add_argument("-n","--name", help="Name contains",default=None)
 parser.add_argument("-c","--type", help="Content Type in.",default=None)
 parser.add_argument("-p","--parent", help="ParentURI starts with.",default=None)
 parser.add_argument("-pf","--parentfolder", help="Parent Folder Name.",default=None)
-parser.add_argument("-d","--days", help="List files older than this number of days",default='-1')
+parser.add_argument("-d","--days", help="List files older or younger than this number of days",default='-1')
+parser.add_argument("-do","--olderoryounger", help="For the date subsetting specify older or younger",choices=['older','younger'],default='older')
 parser.add_argument("-m","--modifiedby", help="Last modified id equals",default=None)
-parser.add_argument("-s","--sortby", help="Sort the output descending by this field",default='modifiedTimeStamp')
+parser.add_argument("-s","--sortby", help="Sort the output by this field",default='modifiedTimeStamp')
+parser.add_argument("-so","--sortorder", help="Sort order",choices=['ascending','descending'],default='descending')
+parser.add_argument("-v","--verbosecsv", help="Verbose CSV(only used with -o=csv) ", action='store_false' )
 parser.add_argument("-o","--output", help="Output Style", choices=['csv','json','simple','simplejson'],default='json')
 parser.add_argument("--debug", action='store_true', help="Debug")
 
 args = parser.parse_args()
 output_style=args.output
-daysolder=args.days
+days=args.days
 modby=args.modifiedby
 sortby=args.sortby
 nameval=args.name
 puri=args.parent
 pfolder=args.parentfolder
 debug=args.debug
-
+verbosecsv=args.verbosecsv
+sortorder=args.sortorder
+olderoryounger=args.olderoryounger
 
 files_result_json=None
 
@@ -73,9 +78,12 @@ if puri !=None and pfolder !=None:
    sys.exit()
 
 # calculate time period for files
-now=dt.today()-td(days=int(daysolder))
+now=dt.today()-td(days=int(days))
 subset_date=now.strftime("%Y-%m-%dT%H:%M:%S")
-datefilter="le(creationTimeStamp,"+subset_date+")"
+
+if olderoryounger=='older':
+  datefilter="le(creationTimeStamp,"+subset_date+")"
+else: datefilter="ge(creationTimeStamp,"+subset_date+")"
 
 # create a list for filter conditions
 filtercond=[]
@@ -94,7 +102,7 @@ delimiter = ','
 if puri!=None: 
    filtercond.append("contains(parentUri,'"+puri+"')")
    completefilter = 'and('+delimiter.join(filtercond)+')'
-   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000"
+   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000"
        
 # process items in folders
 elif pfolder!=None:
@@ -102,7 +110,7 @@ elif pfolder!=None:
    folderid=getfolderid(pfolder)[0]     
    # add the start and end and comma delimit the filter
    completefilter = 'and('+delimiter.join(filtercond)+')'
-   reqval="/folders/folders/"+folderid+"/members?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000"
+   reqval="/folders/folders/"+folderid+"/members?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000"
    
    files_in_folder=callrestapi(reqval,reqtype)
       
@@ -126,18 +134,20 @@ elif pfolder!=None:
    
    filtercond.append("in(id,"+inclause+")")
    completefilter = 'and('+delimiter.join(filtercond)+')'
-   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000"
+   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"g&limit=10000"
    
    
 else:
    completefilter = 'and('+delimiter.join(filtercond)+')'
-   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":descending&limit=10000"
+   reqval="/files/files?filter="+completefilter+"&sortBy="+sortby+":"+sortorder+"&limit=10000"
 
 if debug: print(reqval)   
 
 files_result_json=callrestapi(reqval,reqtype)
-
+if verbosecsv:
-cols=['id','name','contentType','documentType','createdBy','modifiedTimeStamp','size','parentUri']
+   cols=['id','name','contentType','documentType','createdBy','modifiedTimeStamp','size','parentUri']
+else:
+   cols=['id','name','contentType','description','typeDefName','documentType','contentDisposition','fileStatus','searchable','size','creationTimeStamp','createdBy','modifiedBy','modifiedTimeStamp','expirationTimeStamp','encoding','parentUri']
 # print result
 
 if files_result_json == None:

sharedfunctions.py

@@ -387,7 +387,7 @@ def simpleresults(resultdata):
 #   01aug2018  initial development
 #   19dece2018 print  csv in column orderwith only common columns
 
-def csvresults(resultdata,columns=[]):
+def csvresults(resultdata,columns=[],header=1):
 
 
     if 'items' in resultdata:
@@ -421,16 +421,17 @@ def csvresults(resultdata,columns=[]):
             numvals=len(columns)
             z=0
 
-            # print header row of column names
+            if header:
-            for key,val in pairs.items():
+                # print header row of column names
+                for key,val in pairs.items():
 
-                z=z+1
+                    z=z+1
 
-                # seperate with comma except last item
+                    # seperate with comma except last item
-                if z==numvals: sep=''
+                    if z==numvals: sep=''
-                else: sep=','
+                    else: sep=','
 
-                if i==0 and key in columns: print(key,sep,end="")
+                    if i==0 and key in columns: print(key,sep,end="")
 
             print("\n",end="")
 
@@ -519,7 +520,7 @@ def file_accessible(filepath, mode):
 #   22dec2018 add csv columns only relevent for csv output, defaults provided but can be overriden when called
 #   20feb2020 add simplejson output style
 
-def printresult(result,output_style,colsforcsv=["id","name","type","description","creationTimeStamp","modifiedTimeStamp"]):
+def printresult(result,output_style,colsforcsv=["id","name","type","description","creationTimeStamp","modifiedTimeStamp"],header=1):
 
 
     # print rest call results
@@ -530,7 +531,7 @@ def printresult(result,output_style,colsforcsv=["id","name","type","description"
         elif output_style=='simplejson':
             simplejsonresults(result)
         elif output_style=='csv':
-            csvresults(result,columns=colsforcsv)
+            csvresults(result,columns=colsforcsv,header=header)
         else:
             print(json.dumps(result,indent=2))
     else: print(result)