Skip to content

Commit d884e94

Browse files
jorgeboucasjorgeboucas
authored
Merge pull request #128 from mpg-age-bioinformatics/nextcloud
Nextcloud migration
2 parents ee76add + 9cbd509 commit d884e94

File tree

4 files changed

+68
-66
lines changed

4 files changed

+68
-66
lines changed

‎requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ dash-table==5.0.0
2525
dash-table==5.0.0
2626
dash-bootstrap-components==1.3.1
2727
onetimepass==1.0.1
28-
pyocclient==0.6
28+
# pyocclient==0.6
29+
nc_py_api==0.20.2
2930
PyQRCode==1.2.1
3031
python-dotenv==0.21.1
3132
redis==4.5.1

‎utils/owncloud.client.py‎

Lines changed: 62 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
#!/usr/bin/env python3
22

33
import argparse
4+
import os
5+
import sys
6+
import traceback
7+
from datetime import datetime
8+
from nc_py_api import Nextcloud
9+
10+
# --- Argument parsing ---
411
parser = argparse.ArgumentParser(description="Add users and domains to private routes.")
512
parser.add_argument('-v', '--verbose', action="store_true")
613
parser.add_argument('--upload', metavar="<local_folders>", type=str, nargs='*', help="Upload files from folders.", default=["/mpcdf", "/submissions"])
714
parser.add_argument('--download', metavar="<local_folders>", type=str, nargs='*', help="Local folders to download files to.")
8-
parser.add_argument('--target', metavar="<remote_folder>", type=str, nargs='*', help="Owncloud target folder.", default=["mpcdf_submissions", "age_submissions"] )
9-
parser.add_argument('--config', metavar="<config_file>", type=str, nargs='?', help="Config file." )
15+
parser.add_argument('--target', metavar="<remote_folder>", type=str, nargs='*', help="Owncloud/Nextcloud target folder.", default=["mpcdf_submissions", "age_submissions"])
16+
parser.add_argument('--config', metavar="<config_file>", type=str, nargs='?', help="Config file.")
1017
args = parser.parse_args()
1118

12-
import owncloud
13-
import os
14-
import sys
15-
from datetime import datetime
16-
import traceback
17-
19+
# --- Config ---
1820
if args.verbose :
1921
print( datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "Started.")
2022
sys.stdout.flush()
@@ -30,61 +32,58 @@
3032
USER=os.environ.get('OWNCLOUD_USER')
3133
PASS=os.environ.get('OWNCLOUD_PASS')
3234

35+
nc = Nextcloud(nextcloud_url=ADDRESS, nc_auth_user=USER, nc_auth_pass=PASS)
3336

34-
# upload
35-
loggedin=False
36-
if not args.download :
37-
dic=dict(zip( args.upload, args.target ))
38-
for folder in args.upload:
39-
TARGET=dic[folder]
40-
files=os.listdir(folder)
41-
if ( files ) and ( not loggedin ):
42-
oc = owncloud.Client(ADDRESS)
43-
oc.login( USER, PASS )
44-
loggedin=True
45-
for f in files :
46-
try:
47-
response=oc.put_file( os.path.join( TARGET, f ), os.path.join( folder, f ) )
48-
if response:
49-
os.remove(os.path.join( folder, f ))
50-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Uploaded {f}")
51-
sys.stdout.flush()
52-
else:
53-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Could not upload {f}")
54-
sys.stdout.flush()
55-
except Exception as e :
56-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "-- ! EXCEPTION ! --", f, "|", e, traceback.format_exc())
57-
58-
if loggedin:
59-
oc.logout()
37+
# --- Upload ---
38+
if not args.download:
39+
mapping = dict(zip(args.upload, args.target))
40+
for local_folder in args.upload:
41+
remote_folder = mapping[local_folder]
42+
if not os.path.isdir(local_folder):
43+
continue
44+
for filename in os.listdir(local_folder):
45+
local_path = os.path.join(local_folder, filename)
46+
remote_path = f"/{remote_folder}/{filename}"
47+
48+
try:
49+
with open(local_path, "rb") as fp:
50+
nc.files.upload_stream(remote_path, fp, chunk_size=50*1024*1024)
51+
os.remove(local_path)
52+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Uploaded {filename}")
53+
sys.stdout.flush()
54+
except Exception as e:
55+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "-- ! EXCEPTION ! --", filename, "|", e)
56+
traceback.print_exc()
57+
58+
# --- Download ---
59+
if args.download:
60+
mapping = dict(zip(args.download, args.target))
61+
for local_folder in args.download:
62+
remote_folder = mapping[local_folder]
63+
os.makedirs(local_folder, exist_ok=True)
6064

61-
# download
62-
if args.download :
63-
oc = owncloud.Client(ADDRESS)
64-
oc.login( USER, PASS )
65-
dic=dict( zip( args.download, args.target ) )
66-
for folder in args.download:
67-
TARGET=dic[folder]
68-
contents=oc.list( f'{TARGET}/' )
69-
for c in contents:
70-
c_name=c.get_name()
71-
if not os.path.isfile(os.path.join( folder, c_name )):
72-
try:
73-
download=oc.get_file( os.path.join( TARGET, c_name ), local_file=os.path.join( folder, c_name ))
74-
if download:
75-
# print(c.get_path())
76-
oc.delete( os.path.join( TARGET, c_name ) )
77-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Downloaded {c_name}")
78-
sys.stdout.flush()
79-
else:
80-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Could not download {c_name}")
81-
sys.stdout.flush()
82-
except Exception as e :
83-
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "-- ! EXCEPTION ! --", "|", e, traceback.format_exc())
84-
else:
85-
oc.delete( os.path.join( TARGET, c_name ) )
86-
print( datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"File already exists. Removed from source: {c_name}.")
65+
try:
66+
entries = nc.files.listdir(f"/{remote_folder}")
67+
except Exception as e:
68+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Failed to list /{remote_folder}", "|", e)
69+
continue
70+
71+
for entry in entries:
72+
filename = entry.name
73+
local_path = os.path.join(local_folder, filename)
74+
remote_path = f"/{remote_folder}/{filename}"
75+
76+
try:
77+
if not os.path.isfile(local_path):
78+
with open(local_path, "wb") as f_out:
79+
nc.files.download2stream(remote_path, f_out)
80+
nc.files.delete(remote_path)
81+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"Downloaded {filename}")
82+
else:
83+
nc.files.delete(remote_path)
84+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", f"File already exists. Removed from source: {filename}")
85+
sys.stdout.flush()
86+
except Exception as e:
87+
print(datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "-- ! EXCEPTION ! --", "|", e)
88+
traceback.print_exc()
8789
sys.stdout.flush()
88-
oc.logout()
89-
# print( datetime.now().strftime("%d/%m/%Y %H:%M:%S"), "|", "Finished.")
90-
# sys.stdout.flush()

‎utils/requirements.all.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ dash-table==5.0.0
2525
dash-table==5.0.0
2626
dash-bootstrap-components==1.6.0
2727
onetimepass==1.0.1
28-
pyocclient==0.6
28+
# pyocclient==0.6
29+
nc_py_api==0.20.2
2930
PyQRCode==1.2.1
3031
python-dotenv==1.0.1
3132
redis==5.2.0

‎utils/requirements.min.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,8 @@ dash
1212
dash-table
1313
dash-bootstrap-components
1414
onetimepass
15-
pyocclient
15+
# pyocclient
16+
nc_py_api
1617
PyQRCode
1718
python-dotenv
1819
redis

0 commit comments

Comments
 (0)