diff options
| -rw-r--r-- | src/lib/product/api/productSearchApi.js | 31 | ||||
| -rw-r--r-- | src/lib/utils/batchSolrQueries.js | 113 | ||||
| -rw-r--r-- | src/pages/api/shop/search.js | 188 |
3 files changed, 302 insertions, 30 deletions
diff --git a/src/lib/product/api/productSearchApi.js b/src/lib/product/api/productSearchApi.js index a84caa3c..1a6ad36a 100644 --- a/src/lib/product/api/productSearchApi.js +++ b/src/lib/product/api/productSearchApi.js @@ -2,8 +2,37 @@ import _ from 'lodash-contrib'; import axios from 'axios'; const productSearchApi = async ({ query, operation = 'OR' }) => { + // Use POST for large product ID arrays to avoid URL length limits + // GET request URL limit is typically 2KB-8KB; switch to POST if query string is large + const QUERY_SIZE_THRESHOLD = 2000; // Switch to POST if query > 2KB + + if (query.length > QUERY_SIZE_THRESHOLD) { + console.log( + `[productSearchApi] Large query (${query.length} chars), using POST`, + ); + + // Parse query string into object for POST body + const params = new URLSearchParams(query); + const bodyData = { + ...Object.fromEntries(params), + operation, + }; + + const dataProductSearch = await axios.post( + `${process.env.NEXT_PUBLIC_SELF_HOST}/api/shop/search`, + bodyData, + { + headers: { + 'Content-Type': 'application/json', + }, + }, + ); + return dataProductSearch.data; + } + + // Small query, use standard GET request const dataProductSearch = await axios( - `${process.env.NEXT_PUBLIC_SELF_HOST}/api/shop/search?${query}&operation=${operation}` + `${process.env.NEXT_PUBLIC_SELF_HOST}/api/shop/search?${query}&operation=${operation}`, ); return dataProductSearch.data; }; diff --git a/src/lib/utils/batchSolrQueries.js b/src/lib/utils/batchSolrQueries.js new file mode 100644 index 00000000..486f1778 --- /dev/null +++ b/src/lib/utils/batchSolrQueries.js @@ -0,0 +1,113 @@ +/** + * Batch utility functions for handling large product ID arrays in Solr queries + * Prevents URL length limit errors when querying with >100 product IDs + */ + +/** + * Split an array into chunks of specified size + * @param {Array} array - Array to split + * @param {number} size - Chunk size (default: 100) + * @returns {Array<Array>} Array of chunks + */ +export const chunkArray = (array, size = 100) => { + if (!Array.isArray(array) || array.length === 0) return []; + + const chunks = []; + for (let i = 0; i < array.length; i += size) { + chunks.push(array.slice(i, i + size)); + } + return chunks; +}; + +/** + * Build a product ID OR clause for Solr query + * @param {Array<string|number>} ids - Array of product IDs + * @returns {string} Formatted OR clause: "product_id_i:123 OR product_id_i:456..." + */ +export const buildProductIdOrClause = (ids) => { + if (!Array.isArray(ids) || ids.length === 0) { + return '*:*'; + } + return ids.map((id) => `product_id_i:${id}`).join(' OR '); +}; + +/** + * Validate query size to prevent exceeding HTTP limits + * @param {string} query - Query string to validate + * @param {number} maxSize - Maximum allowed size in characters (default: 6000) + * @returns {Object} { valid: boolean, message: string, size: number } + */ +export const validateQuerySize = (query, maxSize = 6000) => { + const size = query.length; + return { + valid: size <= maxSize, + message: + size > maxSize + ? `Query size ${size} exceeds limit of ${maxSize}` + : `Query size ${size} is within limit`, + size, + }; +}; + +/** + * Build batched Solr query parameters for large ID arrays + * Chunks IDs into groups and creates separate OR clauses + * @param {Array<string|number>} ids - Product IDs to query + * @param {number} chunkSize - How many IDs per chunk (default: 100) + * @returns {Array<string>} Array of OR clauses, one per chunk + */ +export const buildBatchedOrClauses = (ids, chunkSize = 100) => { + if (!Array.isArray(ids) || ids.length === 0) { + return ['*:*']; + } + + const chunks = chunkArray(ids, chunkSize); + + if (chunks.length === 1) { + // Single chunk, return standard OR clause + return [buildProductIdOrClause(ids)]; + } + + // Multiple chunks: return OR clauses wrapped with parentheses for combining + return chunks.map((chunk) => `(${buildProductIdOrClause(chunk)})`); +}; + +/** + * Combine multiple OR clauses into a single query (for Solr) + * @param {Array<string>} orClauses - Array of OR clauses + * @returns {string} Combined query with OR between clauses + */ +export const combineOrClauses = (orClauses) => { + if (!Array.isArray(orClauses) || orClauses.length === 0) { + return '*:*'; + } + if (orClauses.length === 1) { + return orClauses[0]; + } + return orClauses.join(' OR '); +}; + +/** + * Merge Solr response documents from multiple queries + * Removes duplicates based on product_id_i + * @param {Array<Array>} responseArrays - Array of response.docs arrays + * @returns {Array} Merged and deduplicated docs + */ +export const mergeSolrResults = (responseArrays) => { + const seen = new Set(); + const merged = []; + + responseArrays.forEach((docs) => { + if (Array.isArray(docs)) { + docs.forEach((doc) => { + const id = doc.product_id_i; + if (id && !seen.has(id)) { + seen.add(id); + merged.push(doc); + } + }); + } + }); + + return merged; +}; diff --git a/src/pages/api/shop/search.js b/src/pages/api/shop/search.js index 1f636f28..8954446a 100644 --- a/src/pages/api/shop/search.js +++ b/src/pages/api/shop/search.js @@ -1,6 +1,11 @@ import { productMappingSolr } from '@/utils/solrMapping'; import axios from 'axios'; import camelcaseObjectDeep from 'camelcase-object-deep'; +import { + chunkArray, + buildProductIdOrClause, + mergeSolrResults, +} from '@/lib/utils/batchSolrQueries'; const escapeSolrQuery = (query) => { if (query == '*') return query; @@ -14,7 +19,26 @@ const escapeSolrQuery = (query) => { .join(' '); }; +/** + * Get parameter value from either GET (req.query) or POST (req.body) + * POST takes precedence if available + */ +const getParam = (req, key, defaultValue = undefined) => { + // POST takes precedence + if (req.body && req.body[key] !== undefined) { + return req.body[key]; + } + // Fallback to GET + if (req.query && req.query[key] !== undefined) { + return req.query[key]; + } + return defaultValue; +}; + export default async function handler(req, res) { + // Support both GET and POST requests + const params = req.method === 'POST' ? req.body : req.query; + const { q = '*', page = 1, @@ -28,9 +52,15 @@ export default async function handler(req, res) { limit = 30, source = '', group = 'true', - } = req.query; + } = params; - let { stock = '' } = req.query; + let { stock = '' } = params; + + // Convert string parameters to appropriate types (for POST requests) + const pageNum = parseInt(page, 10) || 1; + const limitNum = parseInt(limit, 10) || 30; + const priceFromNum = parseFloat(priceFrom) || 0; + const priceToNum = parseFloat(priceTo) || 0; // ============================================================ // [PERBAIKAN] 1. LOGIC KHUSUS COMPARE (PAKAI URLSearchParams) @@ -56,7 +86,7 @@ export default async function handler(req, res) { const params = new URLSearchParams(); params.append('q', qCompare); - params.append('rows', limit); + params.append('rows', limitNum); params.append('wt', 'json'); params.append('indent', 'true'); @@ -218,7 +248,7 @@ export default async function handler(req, res) { // ============================================================ if (source === 'sitemap') { try { - const offset = (page - 1) * limit; + const offset = (pageNum - 1) * limitNum; const parameter = [ 'q=*:*', `rows=${limit}`, @@ -311,7 +341,7 @@ export default async function handler(req, res) { keywords = formattedQuery; } - let offset = (page - 1) * limit; + let offset = (pageNum - 1) * limitNum; let parameter = [ 'facet.field={!ex=brand}manufacture_name_s', @@ -334,9 +364,9 @@ export default async function handler(req, res) { parameter.push(`fq=${encodeURIComponent(f)}`); }); - if (priceFrom > 0 || priceTo > 0) { + if (priceFromNum > 0 || priceToNum > 0) { parameter.push( - `fq=price_tier1_v2_f:[${priceFrom || '*'} TO ${priceTo || '*'}]`, + `fq=price_tier1_v2_f:[${priceFromNum || '*'} TO ${priceToNum || '*'}]`, ); } @@ -371,39 +401,139 @@ export default async function handler(req, res) { fq.map((val) => `fq=${encodeURIComponent(val)}`), ); - // Searchkey - if (req.query.from === 'searchkey') { - const ids = req.query.ids ? req.query.ids.split(',').filter(Boolean) : []; + // Searchkey with batching for large ID arrays + if (params.from === 'searchkey') { + try { + // Extract IDs from either query params (GET) or body (POST) + let ids = []; + + if (req.method === 'POST' && req.body && req.body.ids) { + // POST request: IDs in body + ids = Array.isArray(req.body.ids) + ? req.body.ids + : req.body.ids.split(',').filter(Boolean); + } else if (req.query.ids) { + // GET request: IDs in query params + ids = req.query.ids.split(',').filter(Boolean); + } - const q = ids.map((id) => `product_id_i:${id}`).join(' OR '); + if (ids.length === 0) { + return res.status(400).json({ error: 'No product IDs provided' }); + } - const strictQuery = [ - `q=${encodeURIComponent(q)}`, - `fq=-publish_b:false AND price_tier1_v2_f:[1 TO *] AND product_rating_f:[8 TO *]`, - // `qf=variants_code_t variants_name_t`, - `rows=${limit}`, - `start=${offset}`, - `sort=${paramOrderBy}`, - ]; + console.log(`[SEARCHKEY] Processing ${ids.length} product IDs`); - const solrUrl = - process.env.SOLR_HOST + '/solr/product/select?' + strictQuery.join('&'); + // If less than 100 IDs, use single query + if (ids.length <= 100) { + const q = buildProductIdOrClause(ids); - console.log('[SEARCHKEY FINAL QUERY]', solrUrl); + const strictQuery = [ + `q=${encodeURIComponent(q)}`, + `fq=-publish_b:false AND price_tier1_v2_f:[1 TO *] AND product_rating_f:[8 TO *]`, + `rows=${limitNum}`, + `start=${offset}`, + `sort=${paramOrderBy}`, + ]; - const result = await axios(solrUrl); + const solrUrl = + process.env.SOLR_HOST + + '/solr/product/select?' + + strictQuery.join('&'); - try { - result.data.response.products = productMappingSolr( - result.data.response.docs, + console.log('[SEARCHKEY SINGLE QUERY]', solrUrl); + + const result = await axios(solrUrl); + result.data.response.products = productMappingSolr( + result.data.response.docs, + auth?.pricelist || false, + ); + + delete result.data.response.docs; + result.data = camelcaseObjectDeep(result.data); + + return res.status(200).json(result.data); + } + + // Batch large ID arrays into chunks of 100 + const idChunks = chunkArray(ids, 100); + console.log( + `[SEARCHKEY BATCH] Splitting ${ids.length} IDs into ${idChunks.length} chunks`, + ); + + // Execute all chunk queries in parallel + const batchQueries = idChunks.map((chunk) => { + const q = buildProductIdOrClause(chunk); + const queryParams = [ + `q=${encodeURIComponent(q)}`, + `fq=-publish_b:false AND price_tier1_v2_f:[1 TO *] AND product_rating_f:[8 TO *]`, + `rows=100`, // Request maximum 100 rows per chunk (100 IDs per chunk) + `start=0`, + `sort=${paramOrderBy}`, + ]; + + const solrUrl = + process.env.SOLR_HOST + + '/solr/product/select?' + + queryParams.join('&'); + + console.log( + `[SEARCHKEY BATCH QUERY] Chunk: ${chunk.slice(0, 5).join(',')}...`, + ); + + return axios(solrUrl).catch((error) => { + console.error('[SEARCHKEY BATCH ERROR]', error.message); + throw error; + }); + }); + + // Wait for all queries to complete + const batchResults = await Promise.all(batchQueries); + + // Merge all documents from all chunks, removing duplicates + const allDocs = mergeSolrResults( + batchResults.map((r) => r.data.response.docs), + ); + + console.log( + `[SEARCHKEY MERGE] Merged ${allDocs.length} unique documents from ${batchResults.length} chunks`, + ); + + // Apply pagination on merged results + const paginatedDocs = allDocs.slice(offset, offset + limitNum); + + // Use first result's response structure as template + const templateResponse = batchResults[0].data; + + const mergedResponse = { + ...templateResponse, + response: { + ...templateResponse.response, + numFound: allDocs.length, + start: offset, + rows: limitNum, + docs: paginatedDocs, + }, + responseHeader: { + ...templateResponse.responseHeader, + params: { + ...templateResponse.responseHeader.params, + start: offset, + rows: limitNum, + }, + }, + }; + + mergedResponse.response.products = productMappingSolr( + paginatedDocs, auth?.pricelist || false, ); - delete result.data.response.docs; - result.data = camelcaseObjectDeep(result.data); + delete mergedResponse.response.docs; + mergedResponse.data = camelcaseObjectDeep(mergedResponse); - return res.status(200).json(result.data); + return res.status(200).json(mergedResponse.data); } catch (e) { + console.error('[SEARCHKEY ERROR]', e.message); return res.status(400).json({ error: e.message }); } } |