mirrors/twitter-algorithm
1
0
Fork 0
mirror of https://github.com/twitter/the-algorithm.git synced 2024-12-22 15:35:29 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(navi/dr_transform): add filename:line to file reader error message

Browse source
This commit is contained in:
Pouriya Jahanbakhsh 2023-04-01 02:33:51 +03:30
parent 2dbdfe173c
commit ee5e7fc18d
1 changed files with 5 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
navi/dr_transform/src/util.rs
View file

@ -9,15 +9,17 @@ use std::{
pub fn load_batch_prediction_request_base64(file_name: &str) -> Vec<Vec<u8>> { pub fn load_batch_prediction_request_base64(file_name: &str) -> Vec<Vec<u8>> {
let file = File::open(file_name).expect("could not read file"); let file = File::open(file_name).expect("could not read file");
let mut result = vec![]; let mut result = vec![];
for line in io::BufReader::new(file).lines() { for (mut line_count, line) in io::BufReader::new(file).lines().enumerate() {
line_count += 1;
match base64::decode(line.unwrap().trim()) { match base64::decode(line.unwrap().trim()) {
Ok(payload) => result.push(payload), Ok(payload) => result.push(payload),
Err(err) => println!("error decoding line {err}"), Err(err) => println!("error decoding line {file_name}:{line_count} - {err}"),
} }
} }
println!("result len: {}", result.len()); println!("result len: {}", result.len());
return result result
} }
pub fn save_to_npy<T: npyz::Serialize + AutoSerialize>(data: &[T], save_to: String) { pub fn save_to_npy<T: npyz::Serialize + AutoSerialize>(data: &[T], save_to: String) {
let mut writer = WriteOptions::new() let mut writer = WriteOptions::new()
.default_dtype() .default_dtype()