WJ_Import

Class Import.

Defined (1)

The class is defined in the following location(s).

/classes/WJ_Import.php  
  1. class WJ_Import extends WYSIJA_object { 
  2.  
  3. private $_header_row = array(); 
  4. private $_csv_data = array(); 
  5. private $_match = array(); 
  6. private $_ignore_invalid_date = array(); // specific for date fields 
  7. private $_custom_fields = array(); 
  8. private $_email_key = ''; 
  9. private $_data_to_insert = array(); 
  10. private $_number_list = 0; 
  11. private $_data_numbers = array(); 
  12. private $_line_delimiter = "\n"; 
  13. private $_duplicate_emails_count = array(); // detect the emails that are duplicates in the import file 
  14. private $_csv_array = array(); // containing the csv into an array 
  15. private $_ignored_row_count = 0; // the count of rows we ignore because there was no valid email present 
  16. private $_lines_count = 0; // count the number of valid lines inserted in the DB 
  17. private $_chunks_count = 0; // number of chunks we chopped from the csv 
  18. private $_first_row_is_data = false; // means that there is no header on that csv file 
  19. private $_emails_inserted_in_current_chunk = array(); // array of emails 
  20. private $_csv_file_string = ''; 
  21. private $_data_result = array(); // used for importmatch refactor 
  22. private $_regex_new_field = "/^new_field\|([^\|]+)\|(.+)$/i"; 
  23. private $_field_separators_to_test = array( ', ', ';', "\t" ); 
  24. private $_field_enclosers_to_test = array( '"', '' ); 
  25.  
  26. function __construct() { 
  27. if (!empty($_POST['wysija']['match'])) 
  28. $this->_match = $_POST['wysija']['match']; 
  29. if (!empty($_POST['wysija']['ignore_invalid_date'])) 
  30. $this->_ignore_invalid_date = $_POST['wysija']['ignore_invalid_date']; 
  31. if (!empty($_REQUEST['wysija']['user_list']['list'])) 
  32. $this->_number_list = count($_REQUEST['wysija']['user_list']['list']); 
  33. if (!empty($_POST['firstrowisdata'])) 
  34. $this->_first_row_is_data = true; 
  35. $this->_data_numbers = array('invalid' => array(), 'inserted' => 0, 'valid_email_processed' => 0, 'list_added' => 0, 'list_user_ids' => 0, 'list_list_ids' => $this->_number_list, 'emails_queued' => 0); 
  36.  
  37. /** 
  38. * loading file data passed in a global variable 
  39. * @return type 
  40. */ 
  41. private function _get_temp_file_info() { 
  42. if (!empty($_REQUEST['wysija']['dataImport'])) { 
  43. return (array) json_decode(base64_decode($_REQUEST['wysija']['dataImport']), true); 
  44.  
  45. /** 
  46. * loading the file based on global parameters 
  47. * @return string 
  48. */ 
  49. private function _loading_file_content() { 
  50. // try to access the temporary file created in the previous step 
  51. $this->_csv_data = $this->_get_temp_file_info(); 
  52.  
  53. if ( empty($this->_csv_data['csv']) || !is_string( $this->_csv_data['csv'] ) || preg_match('|[^a-z0-9#_.-]|i', $this->_csv_data['csv']) !== 0 ) { 
  54. $this->error('Error with import file name'); 
  55. return false; 
  56.  
  57. $helper_file = WYSIJA::get('file', 'helper'); 
  58.  
  59. if (!is_string($this->_csv_data['csv']) OR preg_match('|[^a-z0-9#_.-]|i', $this->_csv_data['csv']) !== 0 ) { 
  60. die('Import file error.'); 
  61. $result_file = $helper_file->get($this->_csv_data['csv'], 'import'); 
  62.  
  63. if (!$result_file) { 
  64. $upload_dir = wp_upload_dir(); 
  65. $this->error(sprintf(__('Cannot access CSV file. Verify access rights to this directory "%1$s"', WYSIJA), $upload_dir['basedir']), true); 
  66. return false; 
  67.  
  68. // get the temp csv file 
  69. $this->_csv_file_string = file_get_contents($result_file); 
  70.  
  71. private function _set_custom_fields() { 
  72. $WJ_Field = new WJ_Field(); 
  73. $_custom_fields = $WJ_Field->get_all(); 
  74.  
  75. if(!empty($_custom_fields)) { 
  76. foreach($_custom_fields as $key => &$row) { 
  77. $this->_custom_fields['cf_'.$row->id] = $row; 
  78.  
  79. /** 
  80. * Validate if a column is already matched previously 
  81. * @param string $column_name 
  82. * @return boolean 
  83. */ 
  84. private function _is_column_matched($column_name) { 
  85. if (empty($this->_data_to_insert)) { 
  86. return false; 
  87. $column_names = array_values($this->_data_to_insert); 
  88. return in_array(trim($column_name), $column_names); 
  89. /** 
  90. * match columns together with the csv data based on the data passed 
  91. */ 
  92. private function _match_columns_to_insert() { 
  93.  
  94. // we're going through all of the selected value in each dropdown when importing 
  95. foreach($this->_match as $csv_column_number => $column_in_user_table) { 
  96.  
  97. if (!is_string($column_in_user_table) OR preg_match('|[^a-z0-9#_\|.-]|i', $column_in_user_table) !== 0 ) { 
  98. continue; 
  99. // Reduce matching twice the same column 
  100. if ($this->_is_column_matched($column_in_user_table)) 
  101. continue; 
  102.  
  103. // Ignore `nomatch` columns 
  104. if ($column_in_user_table == 'nomatch') 
  105. continue; 
  106.  
  107. // Check if maybe it's a new field 
  108. preg_match($this->_regex_new_field, $column_in_user_table, $maybe_newfield); 
  109. if ( !empty($maybe_newfield) && in_array($maybe_newfield[1], array('date', 'input')) ) { 
  110. // TODO need to change to WJ_Field I guess when Marco is done moving the files 
  111. // saving a new custom field 
  112. $custom_field = new WJ_Field(); 
  113.  
  114. $custom_field->set(array( 
  115. 'name' => $maybe_newfield[2],  
  116. 'type' => $maybe_newfield[1],  
  117. 'required' => false,  
  118. 'settings' => array( 
  119. 'label' => $maybe_newfield[2],  
  120. )); 
  121.  
  122. $custom_field->save(); 
  123.  
  124. // this is the column name in the database so this is where we need to import that field 
  125. $column_in_user_table = $custom_field->user_column_name(); 
  126. $this->_match[$csv_column_number] = $column_in_user_table; 
  127.  
  128. // keep the match of CSV column number to column key in our database 
  129. // not sure why do we trim the column key ... 
  130. $this->_data_to_insert[$csv_column_number] = trim($column_in_user_table); 
  131.  
  132. // this column is the email column, let's keep track of it for later validation etc.. 
  133. if($column_in_user_table == 'email') { 
  134. $this->_email_key = $csv_column_number; 
  135.  
  136. $this->_set_custom_fields(); 
  137.  
  138. // if the status column is not matched, we make sure that we have an entry for the status column so that we default it to some value on import 
  139. if(!in_array('status', $this->_data_to_insert)) { 
  140. $this->_data_to_insert['status'] = 'status'; 
  141.  
  142. /** 
  143. * build the header of the import query 
  144. * @return type 
  145. */ 
  146. private function _get_import_query_header() { 
  147. return 'INSERT INTO [wysija]user (`' . implode('` , `', $this->_data_to_insert) . '`, `created_at`) VALUES '; 
  148.  
  149. /** 
  150. * @param type $array_csv 
  151. */ 
  152. private function _check_duplicate_emails($array_csv) { 
  153. // look for duplicate emails 
  154. foreach ($array_csv as $keyline => $csv_line) { 
  155. if (isset($csv_line[$this->_email_key])) { 
  156. if (isset($this->_duplicate_emails_count[$csv_line[$this->_email_key]])) { 
  157. $this->_duplicate_emails_count[$csv_line[$this->_email_key]]++; 
  158. //$arra[$keyline] 
  159. } else { 
  160. $this->_duplicate_emails_count[$csv_line[$this->_email_key]] = 1; 
  161. } else { 
  162. //if the record doesn't have the attribute email then we just ignore it 
  163. $this->_ignored_row_count++; 
  164. unset($array_csv[$keyline]); 
  165.  
  166. /** 
  167. * save new column/field match to improve usability the next time our admin 
  168. * import a field with similar headers/columns names 
  169. * @return boolean 
  170. */ 
  171. private function _smart_column_match_recording() { 
  172. if ($this->_first_row_is_data === false) { 
  173. //save the importing fields to be able to match them the next time 
  174. $import_fields = get_option('wysija_import_fields'); 
  175.  
  176. foreach($this->_match as $csv_column_number => $column_in_user_table) { 
  177. if($column_in_user_table != 'nomatch') { 
  178.  
  179. // make a column name key 
  180. $column_name_key = str_replace(array(' ', '-', '_'), '', strtolower($this->_header_row[$csv_column_number])); 
  181. // fill in the array of "csv column name" / "user table column" matches 
  182. $import_fields[$column_name_key] = $column_in_user_table; 
  183.  
  184.  
  185. WYSIJA::update_option('wysija_import_fields' , $import_fields); 
  186. return true; 
  187. return false; 
  188.  
  189. /** 
  190. * import a csv type into wysija's subscribers' table 
  191. * @return type 
  192. */ 
  193. public function import_subscribers() { 
  194.  
  195. // import the contacts 
  196. // 1-check that a list is selected and that there is a csv file pasted 
  197. // 2-save the list if necessary 
  198. // 3-save the contacts and record them for each list selected 
  199. $this->_loading_file_content(); 
  200. // convert the csv file to an array of lines 
  201. $this->_csv_array = $this->_csv_to_array( $this->_csv_file_string , 0 , $this->_csv_data['fsep'] , $this->_csv_data['fenc']); 
  202.  
  203. // check what columns of the csv have been matched together with our subscribers' table 
  204. $this->_match_columns_to_insert(); 
  205.  
  206. $this->_header_row = array_map('trim', $this->_csv_array[0]); 
  207.  
  208. // we process the sql insertion 200 by 200 so that we are safe with the server's memory and cpu 
  209. $csv_chunks = array_chunk($this->_csv_array, 200); 
  210. $this->_csv_array = null; 
  211. $this->_chunks_count = 0; 
  212. $this->_lines_count = 0; 
  213.  
  214. // setting up a timeout value on the sql side to avoid timeout when importing a lot of data apparently. 
  215. global $wpdb; 
  216. $wpdb->query('set session wait_timeout=600'); 
  217.  
  218. // loop and insert the data chunks by chunks 
  219. foreach ($csv_chunks as $key_chunk => $csv_chunk) { 
  220.  
  221. $this->_check_duplicate_emails($csv_chunk); 
  222.  
  223. $result = $this->_import_rows($csv_chunk); 
  224.  
  225. if ($result !== false) 
  226. $this->_chunks_count++; 
  227. else { 
  228. // there was an error we try 3 more times the same chunk and se how it goes 
  229. $try = 0; 
  230. while ($result === false && $try < 3) { 
  231. $result = $this->_import_rows($csv_chunk); 
  232. if ($result !== false) { 
  233. $this->_chunks_count++; 
  234. break; 
  235. $try++; 
  236.  
  237. if ($result === false) { 
  238. $this->error(__('There seems to be an error with the list you\'re trying to import.', WYSIJA), true); 
  239. return false; 
  240. // increment the lines count 
  241. $this->_lines_count += $result; 
  242. // free up some memory 
  243. unset($csv_chunks[$key_chunk]); 
  244.  
  245. // useful the next time we import a file with the same format 
  246. $this->_smart_column_match_recording(); 
  247.  
  248. // refresh the total count of users in wysija 
  249. $helper_user = WYSIJA::get('user', 'helper'); 
  250. $helper_user->refreshUsers(); 
  251.  
  252. // keep only the real duplicate emails unset the unique ones 
  253. // TODO check that this email duplicate function could be a memory sink hole 
  254. // especially that right now we don't use its value 
  255. foreach ($this->_duplicate_emails_count as $email_address => $times_email_in_csv) { 
  256. if ($times_email_in_csv == 1) 
  257. unset($this->_duplicate_emails_count[$email_address]); 
  258.  
  259. // how come we need to do that sometimes? how a lines count could become negative? 
  260. if ($this->_lines_count < 0) 
  261. $this->_lines_count = 0; 
  262.  
  263. // all of these numbers were useful at some point when we were showing more information after an import 
  264. $this->_data_numbers['ignored'] = ($this->_data_numbers['valid_email_processed'] - $this->_data_numbers['inserted']); 
  265. $this->_data_numbers['ignored_list'] = ( ($this->_data_numbers['list_user_ids'] * $this->_data_numbers['list_list_ids']) - $this->_data_numbers['list_added'] ); 
  266.  
  267.  
  268. return $this->_data_numbers; 
  269.  
  270. public function get_duplicate_emails_count() { 
  271. return $this->_duplicate_emails_count; 
  272.  
  273. /** 
  274. * convert a csv string to an array 
  275. * @param type $csv_file_content 
  276. * @param type $rows_to_read 
  277. * @param type $delimiter 
  278. * @param type $enclosure 
  279. * @return array 
  280. */ 
  281. private function _csv_to_array($csv_file_content, $rows_to_read = 0, $delimiter = ', ', $enclosure = '') { 
  282. $data = array(); 
  283.  
  284. if( !(in_array($delimiter, $this->_field_separators_to_test) && in_array($enclosure, $this->_field_enclosers_to_test)) ) { 
  285. $this->error('Unknown csv separators.'); 
  286. return false; 
  287.  
  288. // the new way for splitting a string into an array of lines 
  289. $csv_data_array = explode( $this->_line_delimiter, $csv_file_content ); 
  290.  
  291. $i=1; 
  292. foreach($csv_data_array as $csv_line) { 
  293. if($rows_to_read!=0 && $i> $rows_to_read) return $data; 
  294.  
  295. // str_getcsv only exists in php5 and is a faster and cleaner function than our csv_explode 
  296. if (!function_exists('str_getcsv') || strtoupper(substr(PHP_OS, 0, 3)) === 'WIN') { 
  297. $data[] = $this->_lines_explode($csv_line, $delimiter, $enclosure); 
  298. } else { 
  299. $data[] = str_getcsv($csv_line, $delimiter, $enclosure); 
  300.  
  301.  
  302. $i++; 
  303.  
  304. return $data; 
  305.  
  306. /** 
  307. * explode lines to columns 
  308. * @param type $csv_line 
  309. * @param type $delimiter 
  310. * @param type $enclose 
  311. * @param type $preserve 
  312. * @return type 
  313. */ 
  314. private function _lines_explode($csv_line, $delimiter, $enclose, $preserve = false) { 
  315. $resArr = array(); 
  316. $n = 0; 
  317. if (empty($enclose)) { 
  318. $resArr = explode($delimiter, $csv_line); 
  319. } else { 
  320. $expEncArr = explode($enclose, $csv_line); 
  321. foreach ($expEncArr as $EncItem) { 
  322. if ($n++ % 2) { 
  323. array_push($resArr, array_pop($resArr) . ($preserve ? $enclose : '') . $EncItem . ( $preserve ? $enclose : '')); 
  324. } else { 
  325. $expDelArr = explode($delimiter, $EncItem); 
  326. array_push($resArr, array_pop($resArr) . array_shift($expDelArr)); 
  327. $resArr = array_merge($resArr, $expDelArr); 
  328.  
  329. return $resArr; 
  330.  
  331. /** 
  332. * function processing a chunk of a csv array to import it in the DB 
  333. * @global object $wpdb 
  334. * @param array $csv_chunk 
  335. * @return boolean|string 
  336. */ 
  337. private function _import_rows($csv_chunk) { 
  338.  
  339. global $wpdb; 
  340.  
  341. $this->_emails_inserted_in_current_chunk = array(); 
  342. $time = time(); 
  343. $lines = array(); 
  344. $lines_count = count($csv_chunk); 
  345. $columns_count = count($this->_data_to_insert); 
  346.  
  347. $query = $this->_get_import_query_header(); 
  348.  
  349. // make sure that each line has the right numbers of columns if it doesn't then we can skip it 
  350. foreach ($csv_chunk as $k => $line) { 
  351. if (!(count($line) >= (count($this->_data_to_insert) - 1))) { 
  352. unset($csv_chunk[$k]); 
  353. $lines_count--; 
  354.  
  355. $valid_email_processed = 0; 
  356. $j = 1; 
  357.  
  358. foreach ($csv_chunk as $key_line => $line) { 
  359.  
  360. // if first row is not data but header then we just skip it only on the first chunk 
  361. if ($this->_first_row_is_data === false && $j == 1 && $this->_chunks_count == 0) { 
  362. $j++; 
  363. continue; 
  364.  
  365. $i = 1; 
  366. $values = array(); 
  367.  
  368. // TODO maybe we should check the value of the status column so that if we export a wysija's subscribers' list 
  369. // and import it again in another site then we keep the status 
  370. if (isset($this->_data_to_insert['status'])) 
  371. $line['status'] = 1; 
  372.  
  373. foreach ($line as $key_column => &$value_column) { 
  374.  
  375. // make sure this column is a column we want to insert in our DB 
  376. if (isset($this->_data_to_insert[$key_column])) { 
  377. $column_name = $this->_data_to_insert[$key_column]; 
  378. $original_value_column = $value_column; 
  379. $value_column = $this->_validate_value($column_name, $value_column); 
  380.  
  381. // if it is a a date column, we convert it as a unixtimestamp 
  382. if(isset($this->_custom_fields[$column_name]) && $this->_custom_fields[$column_name]->type == 'date') { 
  383. $value_column = strtotime($value_column); 
  384.  
  385. // this kind of result invalidates the whole row 
  386. if ($value_column === false) { 
  387. // record the invalid row and continue with the loop 
  388. $this->_data_numbers['invalid'][] = $original_value_column; 
  389. unset($csv_chunk[$key_line]); 
  390. $lines_count--; 
  391. continue 2; 
  392. } else { 
  393. // only if this is the email row we record an entry in the recorded emails and the email processed count 
  394. if ($this->_email_key === $key_column) { 
  395. $this->_emails_inserted_in_current_chunk[] = $value_column; 
  396. $valid_email_processed++; 
  397.  
  398. // prepare the query 
  399. $values[] = "'" . esc_sql($value_column, $wpdb->dbh) . "'"; 
  400.  
  401. $values[] = $time; 
  402. $lines[] .= "(" . implode(', ', $values) . ")"; 
  403.  
  404. $query .= implode(', ', $lines); 
  405.  
  406. // update values when the user already exists 
  407. $query .= ' ON DUPLICATE KEY UPDATE '.implode(', ', array_map(array($this, 'process_data_to_insert'), $this->_data_to_insert)); 
  408.  
  409. // replace query to import the subscribers 
  410. $model_wysija = new WYSIJA_model(); 
  411. $import_query = $model_wysija->query($query); 
  412.  
  413. $lines_count = $wpdb->rows_affected; 
  414. $this->_data_numbers['inserted'] += $wpdb->rows_affected; 
  415. $this->_data_numbers['valid_email_processed'] += $valid_email_processed; 
  416.  
  417. if ($import_query === false) { 
  418. $this->error(__('Error when inserting emails.', WYSIJA), true); 
  419. return false; 
  420. $time_now = time(); 
  421. $result_query_import_list = $this->_import_new_users_into_lists($time_now); 
  422.  
  423. $this->_trigger_active_autoresponders($time_now); 
  424.  
  425. if ($result_query_import_list === false) { 
  426. $this->error(__('Error when inserting list.', WYSIJA), true); 
  427. return false; 
  428.  
  429. if ($import_query == 0) 
  430. return '0'; 
  431.  
  432. return $lines_count; 
  433.  
  434. /** 
  435. * used to validate or cast values before importing 
  436. * TODO should we add a type for import of custom fields ? 
  437. * Comment : Marco, feel free to modify entirely 
  438. * @param type $column_name 
  439. * @param type $value 
  440. */ 
  441. function _validate_value($column_name, $value) { 
  442. $value = esc_attr(trim($value)); 
  443.  
  444. switch ($column_name) { 
  445. case 'email': 
  446. return is_email($value); 
  447. break; 
  448. case 'status': 
  449.  
  450. if (in_array(strtolower($value), array('subscribed', 'confirmed', 1, '1', 'true'))) { 
  451. return 1; 
  452. } elseif (in_array(strtolower($value), array('unsubscribed', -1, '-1', 'false'))) { 
  453. return -1; 
  454. } elseif (in_array(strtolower($value), array('unconfirmed', 0, '0'))) { 
  455. return 0; 
  456. else 
  457. return 1; 
  458. break; 
  459. default : 
  460. return $value; 
  461.  
  462. /** 
  463. * take care of active autoresponders retro-activity 
  464. * @param type $time_now 
  465. * @return boolean 
  466. */ 
  467. private function _trigger_active_autoresponders($time_now) { 
  468. $helper_email = WYSIJA::get('email', 'helper'); 
  469. $model_wysija = new WYSIJA_model(); 
  470.  
  471. // list the active auto responders emails 
  472. $active_autoresponders_per_list = $helper_email->get_active_follow_ups(array('email_id', 'params'), true); 
  473.  
  474. if (!empty($active_autoresponders_per_list)) { 
  475. foreach ($_REQUEST['wysija']['user_list']['list'] as $list_id) { 
  476. // checking if this list has a list of follow ups 
  477. if (isset($active_autoresponders_per_list[$list_id])) { 
  478. // for each follow up of that list we queu an email 
  479. foreach ($active_autoresponders_per_list[$list_id] as $key_queue => $follow_up) { 
  480. // insert query per active followup 
  481. $query_queue = 'INSERT IGNORE INTO [wysija]queue (`email_id` , `user_id`, `send_at`) '; 
  482. $query_queue .= ' SELECT ' . (int)$follow_up['email_id'] . ' , B.user_id , ' . ($time_now + $follow_up['delay']); 
  483. $query_queue .= ' FROM [wysija]user_list as B'; 
  484. $query_queue .= ' WHERE B.list_id=' . (int) $list_id . ' AND sub_date=' . $time_now; 
  485.  
  486. $model_wysija->query($query_queue); 
  487.  
  488. $this->_data_numbers['emails_queued'] += $wpdb->rows_affected; 
  489. return true; 
  490. return false; 
  491.  
  492. /** 
  493. * @global type $wpdb 
  494. * @param type $time_now 
  495. * @return type 
  496. */ 
  497. private function _import_new_users_into_lists($time_now) { 
  498. global $wpdb; 
  499. $wpdb->rows_affected = 0; 
  500. $model_wysija = new WYSIJA_model(); 
  501.  
  502. $user_ids = $this->_get_imported_user_ids(); 
  503.  
  504. // insert query per list 
  505. $query = 'INSERT IGNORE INTO [wysija]user_list (`list_id` , `user_id`, `sub_date`) VALUES '; 
  506.  
  507. foreach ($_REQUEST['wysija']['user_list']['list'] as $keyl => $list_id) { 
  508. if (empty($list_id)) 
  509. continue; 
  510. // for each list pre selected go through that process 
  511. foreach ($user_ids as $key => $user_data) { 
  512.  
  513. // inserting each user id to this list 
  514. $query.='(' . (int)$list_id . ' , ' . (int)$user_data['user_id'] . ' , ' . (int)$time_now . ')'; 
  515.  
  516. // if this is not the last row we put a comma for the next row 
  517. if (count($user_ids) > ($key + 1)) { 
  518. $query.=' , '; 
  519.  
  520. // if this is not the last row we put a comma for the next row 
  521. if (count($_REQUEST['wysija']['user_list']['list']) > ($keyl + 1)) { 
  522. $query.=', '; 
  523.  
  524. $result_query = $model_wysija->query($query); 
  525.  
  526. $this->_data_numbers['list_added']+=$wpdb->rows_affected; 
  527. $this->_data_numbers['list_user_ids']+=count($user_ids); 
  528. return $result_query; 
  529.  
  530. /** 
  531. * get a list of user_ids freshly imported 
  532. * @return type 
  533. */ 
  534. private function _get_imported_user_ids() { 
  535. $model_user = WYSIJA::get('user', 'model'); 
  536. // select query to get all of the ids of the emails that have just been inserted 
  537. return $model_user->get(array('user_id'), array('email' => $this->_emails_inserted_in_current_chunk)); 
  538.  
  539.  
  540. /** 
  541. * this save a default list of column matching to ease the import process, this is done only the first time when the 
  542. * field is not populated yet 
  543. */ 
  544. private function _save_default_import_field_match() { 
  545. $import_fields = get_option('wysija_import_fields'); 
  546. if (!$import_fields) { 
  547. $import_fields = array( 
  548. 'fname' => 'firstname',  
  549. 'firstname' => 'firstname',  
  550. 'prenom' => 'firstname',  
  551. 'nom' => 'lastname',  
  552. 'name' => 'lastname',  
  553. 'lastname' => 'lastname',  
  554. 'lname' => 'lastname',  
  555. 'ipaddress' => 'ip',  
  556. 'ip' => 'ip',  
  557. 'addresseip' => 'ip',  
  558. ); 
  559. WYSIJA::update_option('wysija_import_fields', $import_fields); 
  560.  
  561.  
  562. /** 
  563. * put the whole file or posted string into one string and clean up the string that may cause trouble during import 
  564. * @return boolean 
  565. */ 
  566. private function _get_csv_file_cleaned_up() { 
  567. // is it a text import or a file import? 
  568. if($_POST['wysija']['import']['type'] == 'copy') { 
  569. if(!isset($_POST['wysija']['user_list']['csv'])) { 
  570. // memory limit has been reached 
  571. $this->error(__('The list you\'ve pasted is too big for the browser. <strong>Upload the file</strong> instead.', WYSIJA), true); 
  572. return false; 
  573. $this->_csv_file_string = trim(stripslashes($_POST['wysija']['user_list']['csv'])); 
  574.  
  575. }else{ 
  576. // move_uploaded_file($_importfile, $destination); 
  577. $this->_csv_file_string = trim(file_get_contents($_FILES['importfile']['tmp_name'])); 
  578.  
  579. $this->_csv_file_string = str_replace(array("\r", "\n\n", "\n\t\t\n\t\n\t\n\t\n", "\n\t\t\n\t\n\t\n", "\xEF\xBB\xBF", "\n\t\n", "\n(+1)"), array("\n", "\n", "\n ;", "\n", '', ';', ''), $this->_csv_file_string); 
  580.  
  581.  
  582. // this might be gmail recipients(copy-paste from a gmail account) rare paste ... 
  583. if(!preg_match_all('/<([a-z0-9_\'&\.\-\+])+\@(([a-z0-9\-])+\.)+([a-z0-9]{2, 10})+>/i' , $this->_csv_file_string , $matches)) { 
  584. //return false; 
  585. }else{ 
  586.  
  587. if (substr($this->_csv_file_string, -1) != ", ") 
  588. $this->_csv_file_string = trim($this->_csv_file_string) . ', '; 
  589.  
  590.  
  591. if (count($matches[0]) == count($matches)) { 
  592. //this is gmail simple paste 
  593. $this->_csv_file_string = str_replace(array('>, ', '<'), array("\n", ', '), $this->_csv_file_string); 
  594. $this->_csv_file_string = trim($this->_csv_file_string); 
  595.  
  596.  
  597. /** 
  598. * try to figure out the format of that CSV file, which separators and enclosure strings does it use 
  599. * @return boolean 
  600. */ 
  601. private function _run_test_on_csv_file() { 
  602. // try different set of enclosure and separator for the csv which can have different look depending on the data carried 
  603.  
  604. $this->_csv_data['fsep'] = false; 
  605. $this->_csv_data['fenc'] = ''; 
  606. $helper_user = WYSIJA::get('user', 'helper'); 
  607.  
  608. foreach($this->_field_enclosers_to_test as $enclosure) { 
  609. foreach($this->_field_separators_to_test as $fsep) { 
  610.  
  611. // testing different combinations of separator and enclosers 
  612. $this->_csv_array = $this->_csv_to_array( $this->_csv_file_string, 10, $fsep, $enclosure ); 
  613.  
  614. // make sure that our CSV has more than one row and that it has the same number of values in the first row and the second row 
  615. if ( ( count( $this->_csv_array ) > 1 && count( $this->_csv_array[0] ) == count( $this->_csv_array[1] ) ) ) { 
  616. if ( count( $this->_csv_array[0] ) > 1 || $helper_user->validEmail( trim( $this->_csv_array[0][0] ) ) || $helper_user->validEmail( trim( $this->_csv_array[1][0] ) ) ) { 
  617. $this->_csv_data['fsep'] = $fsep; 
  618. $this->_csv_data['fenc'] = $enclosure; 
  619. break(2); 
  620.  
  621. // if we didn't manage to find a separator in that file then it is not a csv file and we come out 
  622. if(empty($this->_csv_data['fsep'])) { 
  623. $this->notice( sprintf( 
  624. "%s <a href='http://support.mailpoet.com/knowledgebase/importing-subscribers-with-a-csv-file/' target='_blank'>%s</a>",  
  625. __("The data you are trying to import doesn't appear to be in the CSV format (Comma Separated Values).", WYSIJA),  
  626. __("Read More", WYSIJA) 
  627. ) ); 
  628. $this->notice(__('The first line of a CSV file should be the column headers : "email", "lastname", "firstname".', WYSIJA)); 
  629. $this->notice(__('The second line of a CSV file should be a set of values : "joeeg@example.com", "Average", "Joe".', WYSIJA)); 
  630.  
  631. $this->notice(__('The two first lines of the file you\'ve uploaded are as follow:', WYSIJA)); 
  632.  
  633. $arraylines = explode("\n", $this->_csv_file_string); 
  634.  
  635. if (empty($arraylines[0])) 
  636. $text = __('Line is empty', WYSIJA); 
  637. else 
  638. $text = $arraylines[0]; 
  639. $this->notice('<strong>' . esc_html($text) . '</strong>'); 
  640.  
  641. if (empty($arraylines[1])) 
  642. $text = __('Line is empty', WYSIJA); 
  643. else 
  644. $text = $arraylines[1]; 
  645. $this->notice('<strong>' . esc_html($text) . '</strong>'); 
  646.  
  647. return false; 
  648.  
  649. // test the size of the file 
  650. $temp_csv_array = $this->_csv_to_array($this->_csv_file_string, 0, $this->_csv_data['fsep'], $this->_csv_data['fenc']); 
  651.  
  652. $this->_data_result['totalrows'] = count($temp_csv_array); 
  653. end($temp_csv_array); 
  654. $this->_data_result['lastrow'] = current($temp_csv_array); 
  655.  
  656.  
  657. /** 
  658. * save the CSV string into a file so that we can use it later in the next step of the process 
  659. * @return boolean|string 
  660. */ 
  661. private function _save_csv_file() { 
  662. // try to make a wysija dir to save the import file 
  663. $helper_file = WYSIJA::get('file', 'helper'); 
  664. $result_dir = $helper_file->makeDir('import'); 
  665. if (!$result_dir) { 
  666. return false; 
  667.  
  668. $file_name = 'import-' . time() . '.csv'; 
  669. $handle = fopen($result_dir . $file_name, 'w'); 
  670. fwrite($handle, $this->_csv_file_string); 
  671. fclose($handle); 
  672.  
  673. return $file_name; 
  674.  
  675.  
  676. /** 
  677. * detect which column is an email one 
  678. */ 
  679. private function _test_csv_emails() { 
  680. $found_email = 0; 
  681. $this->_email_key = array(); 
  682. $helper_user = WYSIJA::get('user', 'helper'); 
  683. foreach ($this->_csv_array as $csv_row) { 
  684. foreach ($csv_row as $key_column => $value_column) { 
  685. if ($helper_user->validEmail(trim($value_column))) { 
  686. $found_email++; 
  687.  
  688. $this->_email_key[$key_column] = $this->_csv_array[0][$key_column]; 
  689.  
  690. $this->_data_result['errormatch'] = false; 
  691. $check_again = __('Check again what you\'re trying to import.', WYSIJA); 
  692. if (($found_email < 1)) { 
  693. $this->error(__('We couldn\'t find any valid addresses in the first 10 rows.', WYSIJA).' '.$check_again, true); 
  694.  
  695. if ((count($this->_csv_array) < 2)) { 
  696. $this->error(__('You\'re import file is not valid.', WYSIJA).' '.$check_again, true); 
  697.  
  698.  
  699. /** 
  700. * function used to scan the CSV before trying to match each column with our own fields 
  701. * @return boolean 
  702. */ 
  703. public function scan_csv_file() { 
  704. $this->_data_result = array(); 
  705.  
  706. // make sure the import match will be as easy as possible with default matches 
  707. if( $this->_save_default_import_field_match() === false) return false; 
  708. // get the csv into a string and check for messy email address etc 
  709. if( $this->_get_csv_file_cleaned_up() === false) return false; 
  710. // find out the format of that CSV file comma semi colon etc .. 
  711. if( $this->_run_test_on_csv_file() === false) return false; 
  712.  
  713. // save the string into a file for later use 
  714. $file_name = $this->_save_csv_file(); 
  715. if ($file_name === false) 
  716. return false; 
  717.  
  718. // look for the email column 
  719. $this->_test_csv_emails(); 
  720.  
  721. // stock the data we will use in the next page 
  722. $this->_data_result['csv'] = $this->_csv_array; 
  723.  
  724. $data_import = array( 
  725. 'csv' => $file_name,  
  726. 'fsep' => $this->_csv_data['fsep'],  
  727. 'fenc' => $this->_csv_data['fenc']); 
  728. $this->_data_result['dataImport'] = base64_encode(json_encode($data_import)); 
  729.  
  730. $this->_data_result['keyemail'] = $this->_email_key; 
  731.  
  732. //test if the first row is data or not 
  733. //test the email column 
  734. foreach ($this->_data_result['keyemail'] as $k) 
  735. $this->_email_key = $k; 
  736.  
  737.  
  738. // test whether the first row is a data row or is a descriptive header row 
  739. $helper_user = WYSIJA::get('user', 'helper'); 
  740. if($helper_user->validEmail( $this->_email_key )) { 
  741. $this->_data_result['firstrowisdata']=true; 
  742. }else{ 
  743. $this->_data_result['totalrows']--; 
  744.  
  745. return $this->_data_result; 
  746.  
  747. private function process_data_to_insert($v) { 
  748. return '`'.$v.'` = VALUES(`'.$v.'`)'; 
  749.