from odoo import models, api, fields from odoo.exceptions import UserError from datetime import datetime, timedelta import logging import pysolr import time from odoo.tools.config import config _logger = logging.getLogger(__name__) _solr = pysolr.Solr('http://10.148.0.5:8983/solr/product/', always_commit=True, timeout=30) _variants_solr = pysolr.Solr('http://10.148.0.5:8983/solr/variants/', always_commit=True, timeout=30) _recommendation_solr = pysolr.Solr('http://10.148.0.5:8983/solr/recommendation/', always_commit=True, timeout=30) # _solr = pysolr.Solr('http://35.219.88.139/solr/product/', always_commit=True, timeout=30) # _variants_solr = pysolr.Solr('http://35.219.88.139/solr/variants/', always_commit=True, timeout=30) # _recommendation_solr = pysolr.Solr('http://35.219.88.139/solr/recommendation/', always_commit=True, timeout=30) # _solr = pysolr.Solr('http://34.101.189.218:8983/solr/product/', always_commit=True, timeout=30) # for development only class ApacheSolr(models.Model): _name = 'apache.solr' _order = 'id desc' def connect(self, schema): env = config.get('solr_env', 'development') url = '' if env == 'development': url = 'http://localhost:8983/solr/' elif env == 'production': url = 'http://34.101.189.218:8983/solr/' return pysolr.Solr(url + schema, always_commit=False, timeout=10) def get_doc(self, schema, id): try: return self.connect(schema).search(f'id:{id}').docs[0] except: return {} def get_docs(self, schema, ids): ids_query = ' OR '.join(ids) return self.connect(schema).search(f'id:({ids_query})').docs def clean_key_docs(self, docs): cleaned_docs = [] for doc in docs: new_dict = self.clean_key_doc(doc) cleaned_docs.append(new_dict) return cleaned_docs def clean_key_doc(self, doc): new_dict = {} for key, value in doc.items(): parts = key.split('_') cleaned_parts = [part for part in parts if part not in ['s', 'i', 'f', 'd', 'b', 'l', 't', 'p']] cleaned_key = '_'.join(cleaned_parts) new_dict[cleaned_key] = value return new_dict def action_commit_collections(self, collections=[]): for collection in collections: self.connect(collection).commit(waitFlush=True, waitSearcher=True) def _update_stock_product_to_solr(self, limit=10000): current_time = datetime.now() delta_time = current_time - timedelta(days=3) current_time = current_time.strftime('%Y-%m-%d %H:%M:%S') delta_time = delta_time.strftime('%Y-%m-%d %H:%M:%S') query = [('__last_update', '<', delta_time),] # query = [('product_variant_id.id', '=', 204903)] stocks = self.env['stock.vendor'].search(query, limit=limit) documents = [] for stock in stocks: stock_total = int(stock.product_variant_id.product_tmpl_id.qty_stock_vendor) # dict_stock = dict({"set": stock_total}) document = { "id": int(stock.product_variant_id.product_tmpl_id.id), "stock_total_f": stock_total } documents.append(document) _logger.info("[SYNC_PRODUCT_STOCK_TO_SOLR] Success Set to solr product %s" % stock.product_variant_id.product_tmpl_id.id) _solr.add(documents, fieldUpdates={'stock_total_f':'set'}) def _update_rating_product_to_solr(self, limit=1000): current_time = datetime.now() delta_time = current_time - timedelta(days=30) current_time = current_time.strftime('%Y-%m-%d %H:%M:%S') delta_time = delta_time.strftime('%Y-%m-%d %H:%M:%S') query = [ '&','&', ('type', '=', 'product'), ('active', '=', True), '|', ('last_calculate_rating', '=', False), ('last_calculate_rating', '<', delta_time), ] # query = [('id', '=', 97942)] templates = self.env['product.template'].search(query, limit=limit) documents=[] for template in templates: rating = int(template.virtual_rating) # dict_rating = dict({'set':rating}) document = { 'id':template.id, 'product_rating_f':rating } documents.append(document) template.last_calculate_rating = current_time _logger.info("[SYNC_PRODUCT_RATING_TO_SOLR] Success Set to solr product %s" % template.id) _logger.info(documents) _solr.add(documents, fieldUpdates={'product_rating_f':'set'}) def _sync_product_to_solr(self, limit=500): start_time = time.time() _logger.info('run sync to solr...') query = ["&","&",("type","=","product"),("active","=",True),"|",("solr_flag","=",0),("solr_flag","=",2)] # query = [('id', '=', 97942)] templates = self.env['product.template'].search(query, limit=limit) documents = [] counter = 0 for template in templates: template_time = time.time() counter += 1 price_excl_after_disc = price_excl = discount = tax = 0 variants_name = variants_code = '' flashsale_data = tier1 = tier2 = tier3 = {} if template.product_variant_count > 1: for variant in template.product_variant_ids: if price_excl_after_disc == 0 or variant._get_website_price_after_disc_and_tax() < price_excl_after_disc: price_excl = variant._get_website_price_exclude_tax() price_excl_after_disc = variant._get_website_price_after_disc_and_tax() discount = variant._get_website_disc(0) tax = variant._get_website_tax() flashsale_data = variant._get_flashsale_price() # add price tiering for base price, discount, and price after discount (tier 1 - 3) tier1 = variant._get_pricelist_tier1() tier2 = variant._get_pricelist_tier2() tier3 = variant._get_pricelist_tier3() else: price_excl_after_disc = price_excl_after_disc price_excl = price_excl discount = discount tax = tax flashsale_data = flashsale_data tier1 = tier1 tier2 = tier2 tier3 = tier3 variants_name += variant.display_name or ''+', ' variants_code += variant.default_code or ''+', ' else: variants_name = template.display_name price_excl = template.product_variant_id._get_website_price_exclude_tax() discount = template.product_variant_id._get_website_disc(0) price_excl_after_disc = template.product_variant_id._get_website_price_after_disc_and_tax() tax = template.product_variant_id._get_website_tax() flashsale_data = template.product_variant_id._get_flashsale_price() tier1 = template.product_variant_id._get_pricelist_tier1() tier2 = template.product_variant_id._get_pricelist_tier2() tier3 = template.product_variant_id._get_pricelist_tier3() category_id = '' category_name = '' for category in template.public_categ_ids: category_id = category.id category_name = category.name document = { 'id': template.id, 'display_name_s': template.display_name, 'name_s': template.name, 'default_code_s': template.default_code or '', 'product_rating_f': template.virtual_rating, 'product_id_i': template.id, 'image_s': self.env['ir.attachment'].api_image('product.template', 'image_512', template.id), 'price_f': price_excl, 'discount_f': discount, 'price_discount_f': price_excl_after_disc, 'tax_f': tax, 'variant_total_i': template.product_variant_count, 'stock_total_f': template.qty_stock_vendor, 'weight_f': template.weight, 'manufacture_id_i': template.x_manufacture.id or 0, 'manufacture_name_s': template.x_manufacture.x_name or '', 'manufacture_name': template.x_manufacture.x_name or '', 'image_promotion_1_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_1', template.x_manufacture.id), 'image_promotion_2_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_2', template.x_manufacture.id), 'category_id_i': category_id or 0, 'category_name_s': category_name or '', 'category_name': category_name or '', 'variants_name_t': variants_name, 'variants_code_t': variants_code, 'search_rank_i': template.search_rank, 'search_rank_weekly_i': template.search_rank_weekly, 'flashsale_id_i': flashsale_data['flashsale_id'] or 0, 'flashsale_tag_s': flashsale_data['flashsale_tag'] or '', 'flashsale_name_s': flashsale_data['flashsale_name'] or '', 'flashsale_base_price_f': flashsale_data['flashsale_base_price'] or 0, 'flashsale_discount_f': flashsale_data['flashsale_discount'] or 0, 'flashsale_price_f': flashsale_data['flashsale_price'] or 0, 'discount_tier1_f': tier1['discount_tier1'] or 0, 'price_tier1_f': tier1['price_tier1'] or 0, 'discount_tier2_f': tier2['discount_tier2'] or 0, 'price_tier2_f': tier2['price_tier2'] or 0, 'discount_tier3_f': tier3['discount_tier3'] or 0, 'price_tier3_f': tier3['price_tier3'] or 0 } documents.append(document) template.solr_flag = 1 # add counter for monitoring _logger.info('[SYNC_PRODUCT_TO_SOLR] {}/{} {:.6f}'.format(counter, limit, time.time() - template_time)) _logger.info('[SYNC_PRODUCT_TO_SOLR] Success add to solr product %s' % template.id) _solr.add(documents) end_time = time.time() _logger.info("[SYNC_PRODUCT_TO_SOLR] Finish task add to solr. Time taken: {:.6f} seconds".format(end_time - start_time)) return True def _test_product_price(self, product_id=228178): product = self.env['product.product'].search([('id', '=', product_id)], limit=1) _logger.info('price incl tax: %s' % product._get_website_price_include_tax()) _logger.info('price excl tax: %s' % product._get_website_price_exclude_tax()) _logger.info('discount : %s' % product._get_website_disc(0)) _logger.info('price after discount : %s' % product._get_website_price_after_disc()) _logger.info('price excl after discount : %s' % product._get_website_price_after_disc_and_tax()) _logger.info('tax : %s' % product._get_website_tax()) def _sync_variants_to_solr(self, limit=500): start_time = time.time() _logger.info('run sync to solr variants...') query = ["&","&","&",("product_tmpl_id.active","=",True),("product_tmpl_id.type","=","product"),("active","=",True),"|",("solr_flag","=",0),("solr_flag","=",2)] # query = [('id', 'in', [184622, 204903])] variants = self.env['product.product'].search(query, limit=limit) documents = [] counter = 0 for variant in variants: template_time = time.time() counter += 1 price_excl_after_disc = price_excl = discount = tax = 0 price_excl = variant._get_website_price_exclude_tax() price_excl_after_disc = variant._get_website_price_after_disc_and_tax() discount = variant._get_website_disc(0) tax = variant._get_website_tax() flashsale_data = variant._get_flashsale_price() category_id = '' category_name = '' for category in variant.product_tmpl_id.public_categ_ids: category_id = category.id category_name = category.name document = { 'id': variant.id, 'display_name_s': variant.display_name, 'name_s': variant.name, 'search_keywords_t': variant.display_name, 'default_code_s': variant.default_code or '', 'product_rating_f': variant.product_tmpl_id.virtual_rating, 'product_id_i': variant.id, 'template_id_i': variant.product_tmpl_id.id, 'image_s': self.env['ir.attachment'].api_image('product.template', 'image_512', variant.product_tmpl_id.id), 'price_f': price_excl, 'discount_f': discount, 'price_discount_f': price_excl_after_disc, 'tax_f': tax, 'stock_total_f': variant.qty_stock_vendor, 'weight_f': variant.product_tmpl_id.weight, 'has_magento_b': variant.has_magento, 'attribute_set_id_i': variant.product_tmpl_id.x_attribute_set_id or 0, 'attribute_set_name_s': variant.product_tmpl_id.x_attribute_set_name or '', 'manufacture_id_i': variant.product_tmpl_id.x_manufacture.id or 0, 'manufacture_name_s': variant.product_tmpl_id.x_manufacture.x_name or '', 'manufacture_name': variant.product_tmpl_id.x_manufacture.x_name or '', 'image_promotion_1_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_1', variant.product_tmpl_id.x_manufacture.id), 'image_promotion_2_s': self.env['ir.attachment'].api_image('x_manufactures', 'image_promotion_2', variant.product_tmpl_id.x_manufacture.id), 'category_id_i': category_id or 0, 'category_name_s': category_name or '', 'category_name': category_name or '', 'search_rank_i': variant.product_tmpl_id.search_rank, 'search_rank_weekly_i': variant.product_tmpl_id.search_rank_weekly, 'flashsale_id_i': flashsale_data['flashsale_id'] or 0, 'flashsale_name_s': flashsale_data['flashsale_name'] or '', 'flashsale_base_price_f': flashsale_data['flashsale_base_price'] or 0, 'flashsale_discount_f': flashsale_data['flashsale_discount'] or 0, 'flashsale_price_f': flashsale_data['flashsale_price'] or 0, } documents.append(document) variant.solr_flag = 1 # add counter for monitoring _logger.info('[SYNC_VARIANTS_TO_SOLR] {}/{} {:.6f}'.format(counter, limit, time.time() - template_time)) _logger.info('[SYNC_VARIANTS_TO_SOLR] Success add to solr variants %s' % variant.id) _variants_solr.add(documents) end_time = time.time() _logger.info("[SYNC_VARIANTS_TO_SOLR] Finish task add to solr. Time taken: {:.6f} seconds".format(end_time - start_time)) return False def _solr_sync_recommendation(self, limit=100): variants = self.env['product.product'].search([('solr_flag', '=', 2)], limit=limit) documents = [] catch = {} for variant in variants: try: document = { 'id': variant.id or 0, 'display_name_s': variant.display_name or '', 'name_s': variant.name or '', 'default_code_s': variant.default_code or '', 'product_rating_f': variant.product_tmpl_id.virtual_rating or 0, 'template_id_i': variant.product_tmpl_id.id or 0, 'active_s': str(variant.active) or 'false', 'type_s': variant.product_tmpl_id.type or '', } variant.write({'solr_flag': 1}) # Ensure the flag is updated correctly documents.append(document) catch = document except Exception as e: _logger.error("Failed to add document to Solr: %s", e) _logger.error("Document data: %s", catch) _recommendation_solr.add(documents) return True