Merge branch 'change/feature/sync-solr' into production

1 files changed, 285 insertions, 0 deletions

diff --git a/indoteknik_custom/models/solr/apache_solr.py b/indoteknik_custom/models/solr/apache_solr.py
new file mode 100644
index 00000000..518280ac
--- /dev/null
+++ b/indoteknik_custom/models/solr/apache_solr.py
@@ -0,0 +1,285 @@
+from odoo import models, api, fields
+from odoo.exceptions import UserError
+from datetime import datetime, timedelta
+import logging
+import pysolr
+import time
+from odoo.tools.config import config
+
+_logger = logging.getLogger(__name__)
+_solr = pysolr.Solr('http://10.148.0.5:8983/solr/product/', always_commit=True, timeout=30)
+_variants_solr = pysolr.Solr('http://10.148.0.5:8983/solr/variants/', always_commit=True, timeout=30)
+# _solr = pysolr.Solr('http://34.101.189.218:8983/solr/product/', always_commit=True, timeout=30) # for development only
+
+
+class ApacheSolr(models.Model):
+    _name = 'apache.solr'
+    _order = 'id desc'
+
+    def connect(self, schema):
+        env = config.get('solr_env', 'development')
+
+        url = ''
+        if env == 'development':
+            url = 'http://192.168.23.5:8983/solr/'
+        elif env == 'production':
+            url = 'http://10.148.0.5:8983/solr/'
+        
+        return pysolr.Solr(url + schema, always_commit=True, timeout=30)
+
+    def get_doc(self, schema, id):
+        try:
+            return self.connect(schema).search(f'id:{id}').docs[0]
+        except:
+            return {}
+        
+    def get_docs(self, schema, ids):
+        ids_query = ' OR '.join(ids)
+        return self.connect(schema).search(f'id:({ids_query})').docs
+        
+    def clean_key_docs(self, docs):
+        cleaned_docs = []
+        for doc in docs:
+            new_dict = self.clean_key_doc(doc)
+            cleaned_docs.append(new_dict)
+        return cleaned_docs
+    
+    def clean_key_doc(self, doc):
+        new_dict = {}
+        for key, value in doc.items():
+            parts = key.split('_')
+            cleaned_parts = [part for part in parts if part not in ['s', 'i', 'f', 'd', 'b', 'l', 't', 'p']]
+            cleaned_key = '_'.join(cleaned_parts)
+            new_dict[cleaned_key] = value
+        return new_dict
+    
+    def _update_stock_product_to_solr(self, limit=10000):
+        current_time = datetime.now()
+        delta_time = current_time - timedelta(days=3)
+
+        current_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
+        delta_time = delta_time.strftime('%Y-%m-%d %H:%M:%S')
+
+        query = [('__last_update', '<', delta_time),]
+        # query = [('product_variant_id.id', '=', 204903)]
+        stocks = self.env['stock.vendor'].search(query, limit=limit)
+        documents = []
+        for stock in stocks:
+            stock_total = int(stock.product_variant_id.product_tmpl_id.qty_stock_vendor)
+            # dict_stock = dict({"set": stock_total})
+            document = {
+                "id": int(stock.product_variant_id.product_tmpl_id.id),
+                "stock_total_f": stock_total
+            }
+            documents.append(document)
+            _logger.info("[SYNC_PRODUCT_STOCK_TO_SOLR] Success Set to solr product %s" % stock.product_variant_id.product_tmpl_id.id)
+        _solr.add(documents, fieldUpdates={'stock_total_f':'set'})
+            
+    def _update_rating_product_to_solr(self, limit=1000):
+        current_time = datetime.now()
+        delta_time = current_time - timedelta(days=30)
+
+        current_time = current_time.strftime('%Y-%m-%d %H:%M:%S')
+        delta_time = delta_time.strftime('%Y-%m-%d %H:%M:%S')
+        query = [
+            '&','&',
+            ('type', '=', 'product'),
+            ('active', '=', True),
+            '|',
+            ('last_calculate_rating', '=', False),
+            ('last_calculate_rating', '<', delta_time),
+        ]
+        # query = [('id', '=', 97942)]
+        templates = self.env['product.template'].search(query, limit=limit)
+        documents=[]
+        for template in templates:
+            rating = int(template.virtual_rating)
+            # dict_rating = dict({'set':rating})
+            document = {
+                'id':template.id,
+                'product_rating_f':rating
+            }
+            documents.append(document)
+            template.last_calculate_rating = current_time
+            _logger.info("[SYNC_PRODUCT_RATING_TO_SOLR] Success Set to solr product %s" % template.id)
+        _logger.info(documents)
+        _solr.add(documents, fieldUpdates={'product_rating_f':'set'})
+
+    def _sync_product_to_solr(self, limit=500):
+        start_time = time.time()
+        _logger.info('run sync to solr...')
+        query = ["&","&",("type","=","product"),("active","=",True),"|",("solr_flag","=",0),("solr_flag","=",2)]
+        # query = [('id', '=', 97942)]
+
+        templates = self.env['product.template'].search(query, limit=limit)
+        documents = []
+        counter = 0
+        for template in templates:
+            template_time = time.time()
+            counter += 1
+            price_excl_after_disc = price_excl = discount = tax = 0
+            variants_name = variants_code = ''
+            flashsale_data = tier1 = tier2 = tier3 = {}
+            if template.product_variant_count > 1:
+                for variant in template.product_variant_ids:
+                    if price_excl_after_disc == 0 or variant._get_website_price_after_disc_and_tax() < price_excl_after_disc:
+                        price_excl = variant._get_website_price_exclude_tax()
+                        price_excl_after_disc = variant._get_website_price_after_disc_and_tax()
+                        discount = variant._get_website_disc(0)
+                        tax = variant._get_website_tax()
+                        flashsale_data = variant._get_flashsale_price()
+                        # add price tiering for base price, discount, and price after discount (tier 1 - 3)
+                        tier1 = variant._get_pricelist_tier1()
+                        tier2 = variant._get_pricelist_tier2()
+                        tier3 = variant._get_pricelist_tier3()
+                    else:
+                        price_excl_after_disc = price_excl_after_disc
+                        price_excl = price_excl
+                        discount = discount
+                        tax = tax
+                        flashsale_data = flashsale_data
+                        tier1 = tier1
+                        tier2 = tier2
+                        tier3 = tier3
+                    variants_name += variant.display_name or ''+', '
+                    variants_code += variant.default_code or ''+', '
+            else:
+                variants_name = template.display_name
+                price_excl = template.product_variant_id._get_website_price_exclude_tax()
+                discount = template.product_variant_id._get_website_disc(0)
+                price_excl_after_disc = template.product_variant_id._get_website_price_after_disc_and_tax()
+                tax = template.product_variant_id._get_website_tax()
+                flashsale_data = template.product_variant_id._get_flashsale_price()
+                tier1 = template.product_variant_id._get_pricelist_tier1()
+                tier2 = template.product_variant_id._get_pricelist_tier2()
+                tier3 = template.product_variant_id._get_pricelist_tier3()
+
+            category_id = ''
+            category_name = ''
+            for category in template.public_categ_ids:
+                category_id = category.id
+                category_name = category.name
+
+            document = {
+                'id': template.id,
+                'display_name_s': template.display_name,
+                'name_s': template.name,
+                'default_code_s': template.default_code or '',
+                'product_rating_f': template.virtual_rating,
+                'product_id_i': template.id,
+                'image_s': self.env['ir.attachment'].api_image('product.template', 'image_512', template.id),
+                'price_f': price_excl,
+                'discount_f': discount,
+                'price_discount_f': price_excl_after_disc,
+                'tax_f': tax,
+                'variant_total_i': template.product_variant_count,
+                'stock_total_f': template.qty_stock_vendor,
+                'weight_f': template.weight,
+                'manufacture_id_i': template.x_manufacture.id or 0,
+                'manufacture_name_s': template.x_manufacture.x_name or '',
+                'manufacture_name': template.x_manufacture.x_name or '',
+                'image_promotion_1_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_1', template.x_manufacture.id),
+                'image_promotion_2_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_2', template.x_manufacture.id),
+                'category_id_i': category_id or 0,
+                'category_name_s': category_name or '',
+                'category_name': category_name or '',
+                'variants_name_t': variants_name,
+                'variants_code_t': variants_code,
+                'search_rank_i': template.search_rank,
+                'search_rank_weekly_i': template.search_rank_weekly,
+                'flashsale_id_i': flashsale_data['flashsale_id'] or 0,
+                'flashsale_tag_s': flashsale_data['flashsale_tag'] or '',
+                'flashsale_name_s': flashsale_data['flashsale_name'] or '',
+                'flashsale_base_price_f': flashsale_data['flashsale_base_price'] or 0,
+                'flashsale_discount_f': flashsale_data['flashsale_discount'] or 0,
+                'flashsale_price_f': flashsale_data['flashsale_price'] or 0,
+                'discount_tier1_f': tier1['discount_tier1'] or 0,
+                'price_tier1_f': tier1['price_tier1'] or 0,
+                'discount_tier2_f': tier2['discount_tier2'] or 0,
+                'price_tier2_f': tier2['price_tier2'] or 0,
+                'discount_tier3_f': tier3['discount_tier3'] or 0,
+                'price_tier3_f': tier3['price_tier3'] or 0
+            }
+            documents.append(document)
+            template.solr_flag = 1
+            # add counter for monitoring
+            _logger.info('[SYNC_PRODUCT_TO_SOLR] {}/{} {:.6f}'.format(counter, limit, time.time() - template_time))
+            _logger.info('[SYNC_PRODUCT_TO_SOLR] Success add to solr product %s' % template.id)
+        _solr.add(documents)
+        end_time = time.time()
+        _logger.info("[SYNC_PRODUCT_TO_SOLR] Finish task add to solr. Time taken: {:.6f} seconds".format(end_time - start_time))
+
+    def _test_product_price(self, product_id=228178):
+        product = self.env['product.product'].search([('id', '=', product_id)], limit=1)
+        _logger.info('price incl tax: %s' % product._get_website_price_include_tax())
+        _logger.info('price excl tax: %s' % product._get_website_price_exclude_tax())
+        _logger.info('discount : %s' % product._get_website_disc(0))
+        _logger.info('price after discount : %s' % product._get_website_price_after_disc())
+        _logger.info('price excl after discount : %s' % product._get_website_price_after_disc_and_tax())
+        _logger.info('tax : %s' % product._get_website_tax())
+
+    def _sync_variants_to_solr(self, limit=500):
+        start_time = time.time()
+        _logger.info('run sync to solr variants...')
+        query = ["&","&","&",("product_tmpl_id.active","=",True),("product_tmpl_id.type","=","product"),("active","=",True),"|",("solr_flag","=",0),("solr_flag","=",2)]
+        # query = [('id', 'in', [184622, 204903])]
+
+        variants = self.env['product.product'].search(query, limit=limit)
+        documents = []
+        counter = 0
+        for variant in variants:
+            template_time = time.time()
+            counter += 1
+
+            price_excl_after_disc = price_excl = discount = tax = 0
+            price_excl = variant._get_website_price_exclude_tax()
+            price_excl_after_disc = variant._get_website_price_after_disc_and_tax()
+            discount = variant._get_website_disc(0)
+            tax = variant._get_website_tax()
+            flashsale_data = variant._get_flashsale_price()
+
+            category_id = ''
+            category_name = ''
+            for category in variant.product_tmpl_id.public_categ_ids:
+                category_id = category.id
+                category_name = category.name
+
+            document = {
+                'id': variant.id,
+                'display_name_s': variant.display_name,
+                'name_s': variant.name,
+                'default_code_s': variant.default_code or '',
+                'product_rating_f': variant.product_tmpl_id.virtual_rating,
+                'product_id_i': variant.id,
+                'template_id_i': variant.product_tmpl_id.id,
+                'image_s': self.env['ir.attachment'].api_image('product.template', 'image_512', variant.product_tmpl_id.id),
+                'price_f': price_excl,
+                'discount_f': discount,
+                'price_discount_f': price_excl_after_disc,
+                'tax_f': tax,
+                'stock_total_f': variant.qty_stock_vendor,
+                'weight_f': variant.product_tmpl_id.weight,
+                'manufacture_id_i': variant.product_tmpl_id.x_manufacture.id or 0,
+                'manufacture_name_s': variant.product_tmpl_id.x_manufacture.x_name or '',
+                'manufacture_name': variant.product_tmpl_id.x_manufacture.x_name or '',
+                'image_promotion_1_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_1', variant.product_tmpl_id.x_manufacture.id),
+                'image_promotion_2_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_2', variant.product_tmpl_id.x_manufacture.id),
+                'category_id_i': category_id or 0,
+                'category_name_s': category_name or '',
+                'category_name': category_name or '',
+                'search_rank_i': variant.product_tmpl_id.search_rank,
+                'search_rank_weekly_i': variant.product_tmpl_id.search_rank_weekly,
+                'flashsale_id_i': flashsale_data['flashsale_id'] or 0,
+                'flashsale_name_s': flashsale_data['flashsale_name'] or '',
+                'flashsale_base_price_f': flashsale_data['flashsale_base_price'] or 0,
+                'flashsale_discount_f': flashsale_data['flashsale_discount'] or 0,
+                'flashsale_price_f': flashsale_data['flashsale_price'] or 0,
+            }
+            documents.append(document)
+            variant.solr_flag = 1
+            # add counter for monitoring
+            _logger.info('[SYNC_VARIANTS_TO_SOLR] {}/{} {:.6f}'.format(counter, limit, time.time() - template_time))
+            _logger.info('[SYNC_VARIANTS_TO_SOLR] Success add to solr variants %s' % variant.id)
+        _variants_solr.add(documents)
+        end_time = time.time()
+        _logger.info("[SYNC_VARIANTS_TO_SOLR] Finish task add to solr. Time taken: {:.6f} seconds".format(end_time - start_time))