Data Upload & Storage Configuration

# Container hierarchy
data/
├── raw/
│   └── ecommerce/          # Raw CSV.gz files
├── processed/
│   └── features/           # Processed features
├── models/
│   └── artifacts/          # Model artifacts
└── logs/
    └── training/           # Training logs

# Install AzCopy (if not installed)
# Windows
winget install Microsoft.AzCopy

# macOS
brew install azcopy

# Linux
wget https://aka.ms/downloadazcopy-v10-linux
tar -xzf downloadazcopy-v10-linux.tar.gz
sudo ./install.sh

# Login to Azure
az login

# Get storage account key
STORAGE_KEY=$(az storage account keys list \
  --resource-group retail-dev-rg \
  --account-name retailforecastdev \
  --query '[0].value' -o tsv)

# Set environment variable
export AZCOPY_STORAGE_KEY=$STORAGE_KEY

# Upload single file
azcopy copy "2019-Jan.csv.gz" \
  "https://retailforecastdev.dfs.core.windows.net/data/raw/ecommerce/2019-Jan.csv.gz" \
  --recursive=false

# Upload multiple files (batch)
azcopy copy "./datasets/" \
  "https://retailforecastdev.dfs.core.windows.net/data/raw/ecommerce/" \
  --recursive=true \
  --include-pattern="2019-*.csv.gz"

# Upload with progress and resume capability
azcopy copy "./datasets/" \
  "https://retailforecastdev.dfs.core.windows.net/data/raw/ecommerce/" \
  --recursive=true \
  --include-pattern="2019-*.csv.gz" \
  --log-level=INFO \
  --overwrite=prompt

# Assign Storage Blob Data Reader to Azure ML Managed Identity
az role assignment create \
  --assignee <aml-workspace-managed-identity-id> \
  --role "Storage Blob Data Reader" \
  --scope "/subscriptions/<subscription-id>/resourceGroups/retail-dev-rg/providers/Microsoft.Storage/storageAccounts/retailforecastdev"

# Assign Storage Blob Data Contributor to ML team
az role assignment create \
  --assignee <ml-team-object-id> \
  --role "Storage Blob Data Contributor" \
  --scope "/subscriptions/<subscription-id>/resourceGroups/retail-dev-rg/providers/Microsoft.Storage/storageAccounts/retailforecastdev"

# List uploaded files
az storage blob list \
  --account-name retailforecastdev \
  --container-name data \
  --prefix "raw/ecommerce/" \
  --output table

# Check file sizes
az storage blob list \
  --account-name retailforecastdev \
  --container-name data \
  --prefix "raw/ecommerce/" \
  --query "[].{Name:name, Size:properties.contentLength}" \
  --output table

# Calculate total size
az storage blob list \
  --account-name retailforecastdev \
  --container-name data \
  --prefix "raw/ecommerce/" \
  --query "[].properties.contentLength" \
  --output tsv | awk '{sum += $1} END {print "Total size:", sum/1024/1024/1024, "GB"}'

# Python script to validate uploaded data
from azure.storage.filedatalake import DataLakeServiceClient
from azure.identity import DefaultAzureCredential
import pandas as pd
import gzip

def validate_uploaded_data():
    credential = DefaultAzureCredential()
    service_client = DataLakeServiceClient(
        account_url="https://retailforecastdev.dfs.core.windows.net",
        credential=credential
    )
    
    file_system_client = service_client.get_file_system_client("data")
    directory_client = file_system_client.get_directory_client("raw/ecommerce")
    
    # List files
    files = list(directory_client.list_paths())
    
    print(f"📁 Found {len(files)} files:")
    total_size = 0
    
    for file_path in files:
        file_client = directory_client.get_file_client(file_path.name)
        properties = file_client.get_file_properties()
        size_mb = properties.size / (1024 * 1024)
        total_size += properties.size
        
        print(f"  ✅ {file_path.name}: {size_mb:.2f} MB")
        
        # Quick validation - try to read first few rows
        try:
            download = file_client.download_file()
            with gzip.open(download, 'rt') as f:
                # Read first 5 lines
                for i, line in enumerate(f):
                    if i >= 5:
                        break
                print(f"    📊 File format valid")
        except Exception as e:
            print(f"    ❌ Error reading file: {e}")
    
    total_gb = total_size / (1024 * 1024 * 1024)
    print(f"\n📈 Total dataset size: {total_gb:.2f} GB")
    
    if total_gb >= 4.0:
        print("✅ Dataset size requirement met (≥4GB)")
    else:
        print("⚠️  Dataset size below requirement (<4GB)")

if __name__ == "__main__":
    validate_uploaded_data()

#!/bin/bash
# upload-dataset.sh

# Configuration
STORAGE_ACCOUNT="retailforecastdev"
RESOURCE_GROUP="retail-dev-rg"
CONTAINER_NAME="data"
DATASET_PATH="./datasets"
TARGET_PATH="raw/ecommerce"

echo "🚀 Starting dataset upload to Azure Storage..."

# Get storage account key
echo "📋 Getting storage account key..."
STORAGE_KEY=$(az storage account keys list \
  --resource-group $RESOURCE_GROUP \
  --account-name $STORAGE_ACCOUNT \
  --query '[0].value' -o tsv)

if [ -z "$STORAGE_KEY" ]; then
    echo "❌ Failed to get storage account key"
    exit 1
fi

# Create container if not exists
echo "📁 Creating container if not exists..."
az storage container create \
  --account-name $STORAGE_ACCOUNT \
  --account-key $STORAGE_KEY \
  --name $CONTAINER_NAME

# Upload files using AzCopy
echo "📤 Uploading files using AzCopy..."
azcopy copy "$DATASET_PATH/" \
  "https://$STORAGE_ACCOUNT.dfs.core.windows.net/$CONTAINER_NAME/$TARGET_PATH/" \
  --recursive=true \
  --include-pattern="2019-*.csv.gz" \
  --log-level=INFO

# Verify upload
echo "✅ Verifying upload..."
az storage blob list \
  --account-name $STORAGE_ACCOUNT \
  --account-key $STORAGE_KEY \
  --container-name $CONTAINER_NAME \
  --prefix "$TARGET_PATH/" \
  --output table

# Calculate total size
echo "📊 Calculating total size..."
TOTAL_SIZE=$(az storage blob list \
  --account-name $STORAGE_ACCOUNT \
  --account-key $STORAGE_KEY \
  --container-name $CONTAINER_NAME \
  --prefix "$TARGET_PATH/" \
  --query "[].properties.contentLength" \
  --output tsv | awk '{sum += $1} END {print sum/1024/1024/1024}')

echo "📈 Total dataset size: ${TOTAL_SIZE} GB"

if (( $(echo "$TOTAL_SIZE >= 4.0" | bc -l) )); then
    echo "✅ Dataset upload completed successfully!"
    echo "✅ Size requirement met (≥4GB)"
else
    echo "⚠️  Warning: Dataset size below requirement (<4GB)"
fi

# storage.tf
resource "azurerm_storage_account" "main" {
  name                     = "retailforecastdev"
  resource_group_name      = azurerm_resource_group.main.name
  location                 = azurerm_resource_group.main.location
  account_tier             = "Standard"
  account_replication_type = "LRS"
  account_kind             = "StorageV2"
  
  # Enable ADLS Gen2
  is_hns_enabled = true
  
  # Security settings
  min_tls_version                 = "TLS1_2"
  allow_nested_items_to_be_public = false
  
  # Enable versioning and soft delete
  versioning_enabled = true
  blob_properties {
    delete_retention_policy {
      days = 7
    }
    versioning_enabled = true
  }
  
  tags = {
    Environment        = "development"
    Project           = "retail-forecast"
    Purpose           = "data-storage"
    DataClassification = "internal"
  }
}

resource "azurerm_storage_container" "data" {
  name                  = "data"
  storage_account_name  = azurerm_storage_account.main.name
  container_access_type = "private"
}