So laden Sie eine Datei mit boto in ein Verzeichnis im S3-Bucket hoch

Ich möchte eine Datei in s3 Bucket mit Python kopieren.

Bsp.: Ich habe Bucket Name = Test. Und im Bucket habe ich 2 Ordner mit den Namen "dump" & "input". Jetzt möchte ich eine Datei mit Python aus dem lokalen Verzeichnis in den S3 "Dump" -Ordner kopieren ... Kann mir jemand helfen?

Versuche dies...

import boto
import boto.s3
import sys
from boto.s3.key import Key

AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''

bucket_name = AWS_ACCESS_KEY_ID.lower() + '-dump'
conn = boto.connect_s3(AWS_ACCESS_KEY_ID,
        AWS_SECRET_ACCESS_KEY)


bucket = conn.create_bucket(bucket_name,
    location=boto.s3.connection.Location.DEFAULT)

testfile = "replace this with an actual filename"
print 'Uploading %s to Amazon S3 bucket %s' % \
   (testfile, bucket_name)

def percent_cb(complete, total):
    sys.stdout.write('.')
    sys.stdout.flush()


k = Key(bucket)
k.key = 'my test file'
k.set_contents_from_filename(testfile,
    cb=percent_cb, num_cb=10)

[UPDATE] Ich bin kein Pythonist, also danke für die Hinweise zu den Importanweisungen. Außerdem würde ich nicht empfehlen, Anmeldeinformationen in Ihren eigenen Quellcode einzufügen. Wenn Sie dies in AWS ausführen, verwenden Sie IAM-Anmeldeinformationen mit Instanzprofilen ( http://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2_instance-profiles.html ) und behalten Sie das gleiche Verhalten bei Verwenden Sie in Ihrer Entwicklungs- / Testumgebung so etwas wie ein Hologramm von AdRoll ( https://github.com/AdRoll/hologram ).

Keine Notwendigkeit, es so kompliziert zu machen:

s3_connection = boto.connect_s3()
bucket = s3_connection.get_bucket('your bucket name')
key = boto.s3.key.Key(bucket, 'some_file.zip')
with open('some_file.zip') as f:
    key.send_file(f)

import boto3

s3 = boto3.resource('s3')
BUCKET = "test"

s3.Bucket(BUCKET).upload_file("your/local/file", "dump/file")

Ich habe dies verwendet und es ist sehr einfach zu implementieren

import tinys3

conn = tinys3.Connection('S3_ACCESS_KEY','S3_SECRET_KEY',tls=True)

f = open('some_file.zip','rb')
conn.upload('some_file.zip',f,'my_bucket')

https://www.smore.com/labs/tinys3/

from boto3.s3.transfer import S3Transfer
import boto3
#have all the variables populated which are required below
client = boto3.client('s3', aws_access_key_id=access_key,aws_secret_access_key=secret_key)
transfer = S3Transfer(client)
transfer.upload_file(filepath, bucket_name, folder_name+"/"+filename)

Laden Sie die Datei innerhalb einer Sitzung mit Anmeldeinformationen auf s3 hoch.

import boto3

session = boto3.Session(
    aws_access_key_id='AWS_ACCESS_KEY_ID',
    aws_secret_access_key='AWS_SECRET_ACCESS_KEY',
)
s3 = session.resource('s3')
# Filename - File to upload
# Bucket - Bucket to upload to (the top level directory under AWS S3)
# Key - S3 object name (can contain subdirectories). If not specified then file_name is used
s3.meta.client.upload_file(Filename='input_file_path', Bucket='bucket_name', Key='s3_output_key')

Dies wird auch funktionieren:

import os 
import boto
import boto.s3.connection
from boto.s3.key import Key

try:

    conn = boto.s3.connect_to_region('us-east-1',
    aws_access_key_id = 'AWS-Access-Key',
    aws_secret_access_key = 'AWS-Secrete-Key',
    # host = 's3-website-us-east-1.amazonaws.com',
    # is_secure=True,               # uncomment if you are not using ssl
    calling_format = boto.s3.connection.OrdinaryCallingFormat(),
    )

    bucket = conn.get_bucket('YourBucketName')
    key_name = 'FileToUpload'
    path = 'images/holiday' #Directory Under which file should get upload
    full_key_name = os.path.join(path, key_name)
    k = bucket.new_key(full_key_name)
    k.set_contents_from_filename(key_name)

except Exception,e:
    print str(e)
    print "error"   

Dies ist ein Drei-Liner. Folgen Sie einfach den Anweisungen in der boto3-Dokumentation .

import boto3
s3 = boto3.resource(service_name = 's3')
s3.meta.client.upload_file(Filename = 'C:/foo/bar/baz.filetype', Bucket = 'yourbucketname', Key = 'baz.filetype')

Einige wichtige Argumente sind:

Parameter:

import boto
from boto.s3.key import Key

AWS_ACCESS_KEY_ID = ''
AWS_SECRET_ACCESS_KEY = ''
END_POINT = ''                          # eg. us-east-1
S3_HOST = ''                            # eg. s3.us-east-1.amazonaws.com
BUCKET_NAME = 'test'        
FILENAME = 'upload.txt'                
UPLOADED_FILENAME = 'dumps/upload.txt'
# include folders in file path. If it doesn't exist, it will be created

s3 = boto.s3.connect_to_region(END_POINT,
                           aws_access_key_id=AWS_ACCESS_KEY_ID,
                           aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
                           host=S3_HOST)

bucket = s3.get_bucket(BUCKET_NAME)
k = Key(bucket)
k.key = UPLOADED_FILENAME
k.set_contents_from_filename(FILENAME)

Mit boto3

import logging
import boto3
from botocore.exceptions import ClientError


def upload_file(file_name, bucket, object_name=None):
    """Upload a file to an S3 bucket

    :param file_name: File to upload
    :param bucket: Bucket to upload to
    :param object_name: S3 object name. If not specified then file_name is used
    :return: True if file was uploaded, else False
    """

    # If S3 object_name was not specified, use file_name
    if object_name is None:
        object_name = file_name

    # Upload the file
    s3_client = boto3.client('s3')
    try:
        response = s3_client.upload_file(file_name, bucket, object_name)
    except ClientError as e:
        logging.error(e)
        return False
    return True

Weitere Informationen: - https://boto3.amazonaws.com/v1/documentation/api/latest/guide/s3-uploading-files.html

Beispiel für das Hochladen eines Ordners wie folgt: Code und S3-Ordnerbild

import boto
import boto.s3
import boto.s3.connection
import os.path
import sys    

# Fill in info on data to upload
# destination bucket name
bucket_name = 'willie20181121'
# source directory
sourceDir = '/home/willie/Desktop/x/'  #Linux Path
# destination directory name (on s3)
destDir = '/test1/'   #S3 Path

#max size in bytes before uploading in parts. between 1 and 5 GB recommended
MAX_SIZE = 20 * 1000 * 1000
#size of parts when uploading in parts
PART_SIZE = 6 * 1000 * 1000

access_key = 'MPBVAQ*******IT****'
secret_key = '11t63yDV***********HgUcgMOSN*****'

conn = boto.connect_s3(
        aws_access_key_id = access_key,
        aws_secret_access_key = secret_key,
        host = '******.org.tw',
        is_secure=False,               # uncomment if you are not using ssl
        calling_format = boto.s3.connection.OrdinaryCallingFormat(),
        )
bucket = conn.create_bucket(bucket_name,
        location=boto.s3.connection.Location.DEFAULT)


uploadFileNames = []
for (sourceDir, dirname, filename) in os.walk(sourceDir):
    uploadFileNames.extend(filename)
    break

def percent_cb(complete, total):
    sys.stdout.write('.')
    sys.stdout.flush()

for filename in uploadFileNames:
    sourcepath = os.path.join(sourceDir + filename)
    destpath = os.path.join(destDir, filename)
    print ('Uploading %s to Amazon S3 bucket %s' % \
           (sourcepath, bucket_name))

    filesize = os.path.getsize(sourcepath)
    if filesize > MAX_SIZE:
        print ("multipart upload")
        mp = bucket.initiate_multipart_upload(destpath)
        fp = open(sourcepath,'rb')
        fp_num = 0
        while (fp.tell() < filesize):
            fp_num += 1
            print ("uploading part %i" %fp_num)
            mp.upload_part_from_file(fp, fp_num, cb=percent_cb, num_cb=10, size=PART_SIZE)

        mp.complete_upload()

    else:
        print ("singlepart upload")
        k = boto.s3.key.Key(bucket)
        k.key = destpath
        k.set_contents_from_filename(sourcepath,
                cb=percent_cb, num_cb=10)

PS: Für mehr Referenz- URL

xmlstr = etree.tostring(listings,  encoding='utf8', method='xml')
conn = boto.connect_s3(
        aws_access_key_id = access_key,
        aws_secret_access_key = secret_key,
        # host = '<bucketName>.s3.amazonaws.com',
        host = 'bycket.s3.amazonaws.com',
        #is_secure=False,               # uncomment if you are not using ssl
        calling_format = boto.s3.connection.OrdinaryCallingFormat(),
        )
conn.auth_region_name = 'us-west-1'

bucket = conn.get_bucket('resources', validate=False)
key= bucket.get_key('filename.txt')
key.set_contents_from_string("SAMPLE TEXT")
key.set_canned_acl('public-read')

Ich habe etwas, das mir etwas mehr Ordnung zu bieten scheint:

import boto3
from pprint import pprint
from botocore.exceptions import NoCredentialsError


class S3(object):
    BUCKET = "test"
    connection = None

    def __init__(self):
        try:
            vars = get_s3_credentials("aws")
            self.connection = boto3.resource('s3', 'aws_access_key_id',
                                             'aws_secret_access_key')
        except(Exception) as error:
            print(error)
            self.connection = None


    def upload_file(self, file_to_upload_path, file_name):
        if file_to_upload is None or file_name is None: return False
        try:
            pprint(file_to_upload)
            file_name = "your-folder-inside-s3/{0}".format(file_name)
            self.connection.Bucket(self.BUCKET).upload_file(file_to_upload_path, 
                                                                      file_name)
            print("Upload Successful")
            return True

        except FileNotFoundError:
            print("The file was not found")
            return False

        except NoCredentialsError:
            print("Credentials not available")
            return False

Hier gibt es drei wichtige Variablen, die BUCKET-Konstante , die Datei_to_upload und den Dateinamen

BUCKET: ist der Name Ihres S3-Buckets

file_to_upload_path: muss der Pfad von der Datei sein, die Sie hochladen möchten

file_name: ist die resultierende Datei und der Pfad in Ihrem Bucket (hier fügen Sie Ordner hinzu oder was auch immer)

Es gibt viele Möglichkeiten, aber Sie können diesen Code in einem anderen Skript wie diesem wiederverwenden

import S3

def some_function():
    S3.S3().upload_file(path_to_file, final_file_name)