Hello,
I'm sorry for the confusion. Let me explain the use case in detail.
The application tries to upload one more files consecutively by invoking upload_file_to_ftp method (snippet below) .
Last year when I was facing the timeout issue, I came across a fix to ftplib by overriding ftplib's ntransfercmd & storbinary commands. It seemed to worked for me at the time (our testing was insufficient). We recently discovered below error logs in out application:
Code: Select all
Traceback (most recent call last):
backupscript.py line 45, in storbinary
return self.voidresp()
File "/usr/lib/python3.8/ftplib.py", line 255, in voidresp
resp = self.getresp()
File "/usr/lib/python3.8/ftplib.py", line 240, in getresp
resp = self.getmultiline()
File "/usr/lib/python3.8/ftplib.py", line 226, in getmultiline
line = self.getline()
File "/usr/lib/python3.8/ftplib.py", line 208, in getline
line = self.file.readline(self.maxline + 1)
AttributeError: 'NoneType' object has no attribute 'readline'
The uploaded backup archives were corrupt.
We have already upgraded to v1.6.7 of FZS to try fixing this, so cannot reproduce logs for previous version (1.5.x)
I tested afresh with two approaches:
1. Without the fix
Code:
Code: Select all
import ftplib
import os
from ftplib import FTP_TLS
def upload_file_to_ftp(srcfile, dst_filename):
ftp_conn = FTP_TLS()
ftp_conn.connect("ftp server url","ftp server port")
ftp_conn.login(user = "username", passwd = 'mypass')
ftp_conn.set_pasv(True)
ftp_conn.prot_p()
try:
folder = 'path to folder'
destpath = os.path.join(folder, os.path.basename(dst_filename))
if folder not in ftp_conn.nlst():
ftp_conn.mkd(folder)
ftp_conn.cwd(folder)
ftp_conn.storbinary("STOR " + os.path.basename(destpath), open(srcfile, "rb"), 1024)
ftp_conn.close()
except ftplib.all_errors as ftp_err:
print("something went wrong")
upload_file_to_ftp(srcfile="/path/to/source/file1", dst_filename="file1")
upload_file_to_ftp(srcfile="/path/to/source/file2", dst_filename="file2")
Log output:
Code: Select all
<Date> Info [Type] Message
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Response] 220-FileZilla Server 1.6.7
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Response] 220 Please visit https://filezilla-project.org/
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Command] AUTH TLS
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Response] 234 Using authentication type TLS.
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Command] USER appserver
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Response] 331 Please, specify the password.
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 [Command] PASS ****
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 230 Login successful.
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Command] PBSZ 0
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 200 PBSZ=0
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Command] PROT P
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 200 Protection level set to P
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Command] TYPE A
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 200 Type set to A
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Command] PASV
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 227 Entering Passive Mode (92,18,27,21,230,221)
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Command] NLST
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 150 About to start data transfer.
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Error] TLS session of data connection not resumed.
<04-05-2023 13:47:26> FTP Session 212 92.18.27.20 appserver [Response] 425 Unable to build data connection: TLS session of data connection not resumed.
No directories or files were created due to the error.
2. With the fix
Code:
Code: Select all
import ftplib
import os
from ftplib import FTP_TLS
class PATCHED_FTP_TLS(FTP_TLS):
"""Explicit FTPS, with shared TLS session"""
def ntransfercmd(self, cmd, rest=None):
import ftplib
conn, size = ftplib.FTP.ntransfercmd(self, cmd, rest)
if self._prot_p:
conn = self.context.wrap_socket(conn,
server_hostname=self.host,
session=self.sock.session) # this is the fix
return conn, size
def storbinary(self, cmd, fp, blocksize=8192, callback=None, rest=None):
"""Store a file in binary mode. A new port is created for you.
Args:
cmd: A STOR command.
fp: A file-like object with a read(num_bytes) method.
blocksize: The maximum data size to read from fp and send over
the connection at once. [default: 8192]
callback: An optional single parameter callable that is called on
each block of data after it is sent. [default: None]
rest: Passed to transfercmd(). [default: None]
Returns:
The response code.
"""
self.voidcmd('TYPE I')
with self.transfercmd(cmd, rest) as conn:
while 1:
buf = fp.read(blocksize)
if not buf:
break
conn.sendall(buf)
if callback:
callback(buf)
# shutdown ssl layer
import ssl
if isinstance(conn, ssl.SSLSocket):
# conn.unwrap()
pass
self.close()
try:
return self.voidresp()
except Exception:
# exc_det = frappe.get_traceback()
# frappe.log_error("Error uploading: %s" % (exc_det))
# send_email(False, "[Exception : FTP]", "Error uploading: {0}".format(exc_det))
return ""
def upload_file_to_ftp(srcfile, dst_filename):
ftp_conn = PATCHED_FTP_TLS()
ftp_conn.connect("ftp server url","ftp server port")
ftp_conn.login(user = "username", passwd = 'mypass')
ftp_conn.set_pasv(True)
ftp_conn.prot_p()
try:
folder = 'path to folder'
destpath = os.path.join(folder, os.path.basename(dst_filename))
if folder not in ftp_conn.nlst():
ftp_conn.mkd(folder)
ftp_conn.cwd(folder)
ftp_conn.storbinary("STOR " + os.path.basename(destpath), open(srcfile, "rb"), 1024)
ftp_conn.close()
except ftplib.all_errors as ftp_err:
print("something went wrong")
upload_file_to_ftp(srcfile="/path/to/source/file1", dst_filename="file1")
upload_file_to_ftp(srcfile="/path/to/source/file2", dst_filename="file2")
Log output:
Code: Select all
<Date> Info [Type] Message
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Response] 220-FileZilla Server 1.6.7
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Response] 220 Please visit https://filezilla-project.org/
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Command] AUTH TLS
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Response] 234 Using authentication type TLS.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Command] USER appserver
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Response] 331 Please, specify the password.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 [Command] PASS ****
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 230 Login successful.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] PBSZ 0
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 200 PBSZ=0
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] PROT P
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 200 Protection level set to P
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] TYPE A
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 200 Type set to A
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] PASV
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 227 Entering Passive Mode (92,18,27,21,235,62)
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] NLST
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 150 Starting data transfer.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 226 Operation successful
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] MKD 20230504_134336
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 257 "/20230504_134336" created successfully.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] CWD 20230504_134336
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 250 CWD command successful
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] TYPE I
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 200 Type set to I
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] PASV
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 227 Entering Passive Mode (92,18,27,21,242,38)
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Command] STOR 20230504_134336-myapp-database.sql.gz
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Response] 150 About to start data transfer.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Error] GnuTLS error -110 in gnutls_record_recv: The TLS connection was non-properly terminated.
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Status] Client did not properly shut down TLS connection
<04-05-2023 13:43:38> FTP Session 208 92.18.27.20 appserver [Error] Control channel closed with error from source 0. Reason: ECONNABORTED - Connection aborted.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Response] 220-FileZilla Server 1.6.7
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Response] 220 Please visit https://filezilla-project.org/
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Command] AUTH TLS
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Response] 234 Using authentication type TLS.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Command] USER appserver
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Response] 331 Please, specify the password.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 [Command] PASS ****
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 230 Login successful.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] PBSZ 0
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 200 PBSZ=0
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] PROT P
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 200 Protection level set to P
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] TYPE A
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 200 Type set to A
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] PASV
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 227 Entering Passive Mode (92,18,27,21,217,117)
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] NLST
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 150 About to start data transfer.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 226 Operation successful
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] CWD 20230504_134336
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 250 CWD command successful
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] TYPE I
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 200 Type set to I
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] PASV
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 227 Entering Passive Mode (92,18,27,21,244,78)
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Command] STOR 20230504_134336-myapp-files.tar
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Response] 150 Starting data transfer.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Error] GnuTLS error -110 in gnutls_record_recv: The TLS connection was non-properly terminated.
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Status] Client did not properly shut down TLS connection
<04-05-2023 13:43:39> FTP Session 209 92.18.27.20 appserver [Error] Control channel closed with error from source 0. Reason: ECONNABORTED - Connection aborted.
gunzip output
Code: Select all
$ gunzip 20230504_124937-myapp-database.sql.gz
gzip: 20230504_124937-myapp-database.sql.gz: unexpected end of file