1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
|
# -*- coding: utf-8 -*-
"""
Tests for various autodetection magics for CSV imports
"""
import codecs
from odoo.tests import common
class ImportCase(common.TransactionCase):
def _make_import(self, contents):
return self.env['base_import.import'].create({
'res_model': 'base_import.tests.models.complex',
'file_name': 'f',
'file_type': 'text/csv',
'file': contents,
})
class TestEncoding(ImportCase):
"""
create + parse_preview -> check result options
"""
def _check_text(self, text, encodings, **options):
options.setdefault('quoting', '"')
options.setdefault('separator', '\t')
test_text = "text\tnumber\tdate\tdatetime\n%s\t1.23.45,67\t\t\n" % text
for encoding in ['utf-8', 'utf-16', 'utf-32', *encodings]:
if isinstance(encoding, tuple):
encoding, es = encoding
else:
es = [encoding]
preview = self._make_import(
test_text.encode(encoding)).parse_preview(dict(options))
self.assertIsNone(preview.get('error'))
guessed = preview['options']['encoding']
self.assertIsNotNone(guessed)
self.assertIn(
codecs.lookup(guessed).name, [
codecs.lookup(e).name
for e in es
]
)
def test_autodetect_encoding(self):
""" Check that import preview can detect & return encoding
"""
self._check_text("Iñtërnâtiônàlizætiøn", [('iso-8859-1', ['iso-8859-1', 'iso-8859-2'])])
self._check_text("やぶら小路の藪柑子。海砂利水魚の、食う寝る処に住む処、パイポパイポ パイポのシューリンガン。", ['eucjp', 'shift_jis', 'iso2022_jp'])
self._check_text("대통령은 제4항과 제5항의 규정에 의하여 확정된 법률을 지체없이 공포하여야 한다, 탄핵의 결정.", ['euc_kr', 'iso2022_kr'])
# + control in widget
def test_override_detection(self):
""" ensure an explicitly specified encoding is not overridden by the
auto-detection
"""
s = "Iñtërnâtiônàlizætiøn".encode('utf-8')
r = self._make_import(b'text\n' + s)\
.parse_preview({
'quoting': '"',
'separator': '\t',
'encoding': 'iso-8859-1',
})
self.assertIsNone(r.get('error'))
self.assertEqual(r['options']['encoding'], 'iso-8859-1')
self.assertEqual(r['preview'], [['text'], [s.decode('iso-8859-1')]])
class TestFileSeparator(ImportCase):
def setUp(self):
super().setUp()
self.imp = self._make_import(
"""c|f
a|1
b|2
c|3
d|4
""")
def test_explicit_success(self):
r = self.imp.parse_preview({
'separator': '|',
'headers': True,
'quoting': '"',
})
self.assertIsNone(r.get('error'))
self.assertEqual(r['headers'], ['c', 'f'])
self.assertEqual(r['preview'], [
['a', '1'],
['b', '2'],
['c', '3'],
['d', '4'],
])
self.assertEqual(r['options']['separator'], '|')
def test_explicit_fail(self):
""" Don't protect user against making mistakes
"""
r = self.imp.parse_preview({
'separator': ',',
'headers': True,
'quoting': '"',
})
self.assertIsNone(r.get('error'))
self.assertEqual(r['headers'], ['c|f'])
self.assertEqual(r['preview'], [
['a|1'],
['b|2'],
['c|3'],
['d|4'],
])
self.assertEqual(r['options']['separator'], ',')
def test_guess_ok(self):
r = self.imp.parse_preview({
'separator': '',
'headers': True,
'quoting': '"',
})
self.assertIsNone(r.get('error'))
self.assertEqual(r['headers'], ['c', 'f'])
self.assertEqual(r['preview'], [
['a', '1'],
['b', '2'],
['c', '3'],
['d', '4'],
])
self.assertEqual(r['options']['separator'], '|')
def test_noguess(self):
""" If the guesser has no idea what the separator is, it defaults to
"," but should not set that value
"""
imp = self._make_import('c\na\nb\nc\nd')
r = imp.parse_preview({
'separator': '',
'headers': True,
'quoting': '"',
})
self.assertIsNone(r.get('error'))
self.assertEqual(r['headers'], ['c'])
self.assertEqual(r['preview'], [
['a'],
['b'],
['c'],
['d'],
])
self.assertEqual(r['options']['separator'], '')
class TestNumberSeparators(common.TransactionCase):
def test_parse_float(self):
w = self.env['base_import.import'].create({
'res_model': 'base_import.tests.models.float',
})
data = w._parse_import_data(
[
['1.62'], ['-1.62'], ['+1.62'], [' +1.62 '], ['(1.62)'],
["1'234'567,89"], ["1.234.567'89"]
],
['value'], {}
)
self.assertEqual(
[d[0] for d in data],
['1.62', '-1.62', '+1.62', '+1.62', '-1.62',
'1234567.89', '1234567.89']
)
|
