Product Import Strategies in Magento 2 – Part 4: REST API and Microservice Integration

    Magendoo Insights

    Introduction

    In this final installment of our series, we’ll explore implementing product imports using Magento 2’s REST API with a microservice architecture. This approach provides real-time feedback, granular control, and the ability to handle complex product relationships.

    System Architecture

    Implementation

    1. Microservice Core

    // Product Import Service (Node.js/TypeScript)
    import axios, { AxiosInstance } from 'axios';
    import { Queue } from './queue';
    import { ImageProcessor } from './image-processor';
    
    interface MagentoConfig {
        baseUrl: string;
        token: string;
        batchSize: number;
    }
    
    class ProductImportService {
        private readonly client: AxiosInstance;
        private readonly queue: Queue;
        private readonly imageProcessor: ImageProcessor;
    
        constructor(config: MagentoConfig) {
            this.client = axios.create({
                baseURL: config.baseUrl,
                headers: {
                    'Authorization': `Bearer ${config.token}`,
                    'Content-Type': 'application/json'
                }
            });
            this.queue = new Queue(config.batchSize);
            this.imageProcessor = new ImageProcessor();
        }
    
        async importProducts(products: Product[]): Promise<ImportResult> {
            try {
                // Queue products in batches
                await this.queue.addBatch(products);
    
                // Process queue
                while (!this.queue.isEmpty()) {
                    const batch = await this.queue.getNext();
                    await this.processBatch(batch);
                }
    
                return this.generateReport();
            } catch (error) {
                this.handleError(error);
            }
        }
    
        private async processBatch(products: Product[]): Promise<void> {
            for (const product of products) {
                try {
                    // Process images first
                    const processedImages = await this.imageProcessor.process(product.images);
    
                    // Prepare product data with processed image URLs
                    const productData = this.prepareProductData(product, processedImages);
    
                    // Check if product exists
                    const exists = await this.checkProductExists(product.sku);
    
                    if (exists) {
                        await this.updateProduct(product.sku, productData);
                    } else {
                        await this.createProduct(productData);
                    }
                } catch (error) {
                    this.logError(product.sku, error);
                }
            }
        }
    
        private async createProduct(productData: any): Promise<void> {
            await this.client.post('/rest/V1/products', {
                product: productData
            });
        }
    
        private async updateProduct(sku: string, productData: any): Promise<void> {
            await this.client.put(`/rest/V1/products/${sku}`, {
                product: productData
            });
        }
    }

    2. Product Data Transformer

    class ProductTransformer {
        transformForMagento(rawProduct: RawProduct): MagentoProduct {
            return {
                sku: rawProduct.sku,
                name: rawProduct.name,
                price: rawProduct.price,
                status: 1,
                visibility: 4,
                type_id: 'simple',
                attribute_set_id: 4,
                extension_attributes: this.getExtensionAttributes(rawProduct),
                custom_attributes: this.getCustomAttributes(rawProduct)
            };
        }
    
        private getExtensionAttributes(product: RawProduct): any {
            return {
                website_ids: [1],
                category_links: this.getCategoryLinks(product.categories),
                stock_item: {
                    qty: product.quantity,
                    is_in_stock: product.quantity > 0
                }
            };
        }
    
        private getCustomAttributes(product: RawProduct): any[] {
            const attributes = [];
    
            // Handle each custom attribute
            if (product.description) {
                attributes.push({
                    attribute_code: 'description',
                    value: product.description
                });
            }
    
            // Add more custom attributes as needed
            return attributes;
        }
    }

    3. Image Processing Service

    class ImageProcessor {
        private readonly allowedTypes = ['jpg', 'jpeg', 'png', 'webp'];
        private readonly maxSize = 2000; // pixels
    
        async process(images: ProductImage[]): Promise<ProcessedImage[]> {
            const processed: ProcessedImage[] = [];
    
            for (const image of images) {
                try {
                    // Download image
                    const buffer = await this.downloadImage(image.url);
    
                    // Validate image
                    await this.validateImage(buffer);
    
                    // Process image
                    const optimized = await this.optimize(buffer);
    
                    // Upload to storage
                    const url = await this.upload(optimized, image.label);
    
                    processed.push({
                        original: image.url,
                        processed: url,
                        label: image.label,
                        position: image.position,
                        types: image.types || ['image']
                    });
                } catch (error) {
                    this.logImageError(image.url, error);
                }
            }
    
            return processed;
        }
    
        private async optimize(buffer: Buffer): Promise<Buffer> {
            // Implement image optimization logic
            // Example: using sharp for image processing
            return sharp(buffer)
                .resize(this.maxSize, this.maxSize, {
                    fit: 'inside',
                    withoutEnlargement: true
                })
                .webp({ quality: 85 })
                .toBuffer();
        }
    }

    4. Queue Management

    class ImportQueue {
        private readonly batchSize: number;
        private queue: Product[][];
        private processing: boolean;
    
        constructor(batchSize: number) {
            this.batchSize = batchSize;
            this.queue = [];
            this.processing = false;
        }
    
        async addBatch(products: Product[]): Promise<void> {
            const batches = this.createBatches(products);
            this.queue.push(...batches);
        }
    
        private createBatches(products: Product[]): Product[][] {
            const batches: Product[][] = [];
    
            for (let i = 0; i < products.length; i += this.batchSize) {
                batches.push(products.slice(i, i + this.batchSize));
            }
    
            return batches;
        }
    
        async processQueue(handler: (batch: Product[]) => Promise<void>): Promise<void> {
            if (this.processing) {
                return;
            }
    
            this.processing = true;
    
            try {
                while (this.queue.length > 0) {
                    const batch = this.queue.shift();
                    if (batch) {
                        await handler(batch);
                    }
                }
            } finally {
                this.processing = false;
            }
        }
    }

    API Integration Patterns

    1. Error Handling and Retry Logic

    class RetryManager {
        private readonly maxRetries: number;
        private readonly backoffMs: number;
    
        constructor(maxRetries: number = 3, backoffMs: number = 1000) {
            this.maxRetries = maxRetries;
            this.backoffMs = backoffMs;
        }
    
        async retry<T>(operation: () => Promise<T>): Promise<T> {
            let lastError: Error;
    
            for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
                try {
                    return await operation();
                } catch (error) {
                    lastError = error;
    
                    if (!this.isRetryable(error) || attempt === this.maxRetries) {
                        throw error;
                    }
    
                    await this.delay(attempt);
                }
            }
    
            throw lastError;
        }
    
        private isRetryable(error: any): boolean {
            // Define retry conditions
            return error.response?.status >= 500 || 
                   error.code === 'ECONNRESET' ||
                   error.code === 'ETIMEDOUT';
        }
    }

    2. Bulk Operations

    class BulkOperationManager {
        private readonly client: AxiosInstance;
    
        async executeBulkOperation(operations: Operation[]): Promise<BulkResult> {
            const bulkRequest = {
                operations: operations.map(op => ({
                    entity: op.entity,
                    action: op.action,
                    data: op.data
                }))
            };
    
            const response = await this.client.post('/rest/async/bulk/V1/products', bulkRequest);
            return this.trackBulkOperation(response.data.bulk_uuid);
        }
    
        private async trackBulkOperation(uuid: string): Promise<BulkResult> {
            while (true) {
                const status = await this.getBulkStatus(uuid);
    
                if (status.is_complete) {
                    return this.processBulkResult(status);
                }
    
                await new Promise(resolve => setTimeout(resolve, 5000));
            }
        }
    }

    Monitoring and Logging

    class ImportMonitor {
        private readonly metrics: MetricsClient;
        private readonly logger: Logger;
    
        recordMetrics(result: ImportResult): void {
            this.metrics.gauge('import.products.total', result.total);
            this.metrics.gauge('import.products.success', result.success);
            this.metrics.gauge('import.products.failed', result.failed);
            this.metrics.timing('import.duration', result.duration);
        }
    
        async monitorHealth(): Promise<void> {
            const metrics = await this.gatherMetrics();
    
            if (metrics.errorRate > this.threshold) {
                await this.alertTeam({
                    type: 'error_rate',
                    metrics: metrics,
                    timestamp: new Date()
                });
            }
        }
    }

    Best Practices

    1. API Usage

    • Implement rate limiting
    • Use bulk operations when possible
    • Cache API responses
    • Handle token expiration

    2. Error Handling

    • Implement circuit breakers
    • Use exponential backoff
    • Log detailed error information
    • Handle partial success scenarios

    3. Performance

    • Process images asynchronously
    • Use connection pooling
    • Implement request batching
    • Cache frequently accessed data

    Advantages

    1. Real-time Processing
    • Immediate feedback
    • Error handling
    • Progress tracking
    1. Scalability
    • Independent scaling
    • Resource optimization
    • Load distribution
    1. Flexibility
    • Custom business logic
    • Multiple data sources
    • Easy integration

    Limitations

    1. Complexity
    • API maintenance
    • Token management
    • Version compatibility
    1. Performance
    • API rate limits
    • Network latency
    • Resource constraints

    When to Use This Approach

    This approach is ideal when:

    • Real-time processing is required
    • Complex validation rules exist
    • Integration with multiple systems is needed
    • Custom business logic is required

    Conclusion

    The REST API with microservice approach offers:

    • Real-time processing capabilities
    • Granular control over the import process
    • Flexibility in handling complex scenarios
    • Scalability for large operations

    Choose this approach when you need tight integration with Magento 2 and want to maintain control over the entire import process.


    This concludes our series on Product Import Strategies in Magento 2. Each approach has its strengths, and the choice depends on your specific requirements and constraints.

    magento2 #api #microservices #ecommerce #nodejs #typescript #programming

    Leave a Reply

    Your email address will not be published. Required fields are marked *