Tuesday, December 13, 2022

[Python] Push file to Azure Blob form Local HD or from Public URL

 Sample Python code, python version should be 3.7 or higher.



# -*- coding: utf-8 -*-
#!/usr/bin/python
from __future__ import print_function
import pymysql.cursors 
import sys
import re
#pip3 install azure-storage-blob azure-identity
import os, uuid
from azure.identity import DefaultAzureCredential
from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
import time


def Upload2AzureFromURL(NewFileName,FileURL,BibID):
    try:
        connectionString = "DefaultEndpointsProtocol=https;AccountName=**********;AccountKey=*****;EndpointSuffix=core.windows.net"
        containerName = "newcontainer"
        blob_service_client = BlobServiceClient.from_connection_string(connectionString)
        blob_client = blob_service_client.get_blob_client(container=containerName, blob=NewFileName)
        blob_client.start_copy_from_url(FileURL,metadata={"BibID":str(BibID)})
        for i in range(10):
            props = blob_client.get_blob_properties()
            status = props.copy.status
            print("Copy status: " + status)
            if status == "success":
                # Copy finished
                print("success...")
                break
            time.sleep(10)

        if status != "success":
            # if not finished after 100s, cancel the operation
            props = blob_client.get_blob_properties()
            print(props.copy.status)
            copy_id = props.copy.id
            copied_blob.abort_copy(copy_id)
            props = copied_blob.get_blob_properties()
            print(props.copy.status)



    except Exception as ex:
        print('Exception:')
        print(ex)

def Upload2AzureFromLocalHD(NewFileName,FullFilePath,BibID):
    try:
        connectionString = "DefaultEndpointsProtocol=https;AccountName=********;AccountKey=****;EndpointSuffix=core.windows.net"
        containerName = "newcontainer"
        blob_service_client = BlobServiceClient.from_connection_string(connectionString)
        blob_client = blob_service_client.get_blob_client(container=containerName, blob=NewFileName)
        blob_client.set_blob_metadata(metadata={"BibID":str(BibID)})
        with open(file=FullFilePath, mode="rb") as data:
            blob_client.upload_blob(data)



    except Exception as ex:
        print('Exception:')
        print(ex)






mrqoom_db = pymysql.connect(host="**********",    # your host, usually localhost
                     user="***********",         # your username
                     passwd="***********",   # your password
                     db="***",  
                     charset='utf8')


mrqoom_db.autocommit(True)
mrqoom_cur = mrqoom_db.cursor(pymysql.cursors.DictCursor)
mrqoom_cur.execute("SET session group_concat_max_len=30000;")

mrqoom_cur.execute("""
                    SELECT id,BibID,url FROM `files` 
                   """) 

try:
    MarcRecords=mrqoom_cur.fetchall()
    for row in MarcRecords:
        try:
            print('Try Download FT for ID: '+str(row["id"])+'     '+str(row["url"]))
            NewFileName=str(row["url"]).rsplit('/', 1)[-1]
            Upload2AzureFromURL(NewFileName,str(row["url"]),str(row["BibID"]))
        except Exception as e:
            print ("Fail!")
            print (str(e))
except Exception as e:
    print ("Fail!")
    print (str(e))

mrqoom_cur.close()
mrqoom_db.close()

print ("============= END ====================")