file io - C# How should I save a large database of information? -
i wrote code serialize object containing data, save it, , load next time program run. (using binaryformater)
however program takes insanely long time load data (at 100 entrys, takes around 15-30 seconds), , need file store more @ around 300,000 entrys. save time isn't great either faster loading. i'd know sort of options there saving , loading database's , differences between them. have read xml pretty slow, know how lazy loading works (load needed), , file formats can applied to.
here code saves , repopulates data:
static void saveasbinary(string fullpath, object data) { // create new, empty data file. string filename = fullpath; if (file.exists(filename)) { console.writeline(filename + @" exists!"); if (file.exists(filename + ".bak")) { file.delete(filename + ".bak"); } file.move(filename, filename + ".bak"); } try { binaryformatter bf = new binaryformatter(); filestream fs = new filestream(filename, filemode.createnew); bf.serialize(fs, data); fs.flush(true); fs.dispose(); } catch (exception ex) { console.writeline("exception: " + ex.message); } } private void populatedatabase(savedatabaseindex db) { if (last_page < cur_page) db.current_page = -1; db.urllist = urllist; db.firstentryonsave = firstentryonload; foreach (string url in urllist) { eh_entry ent = getentry(url); details_panesavedata detpane = new details_panesavedata { rating = ent.detailspanel.rating, uploader = ent.detailspanel.uploader, category = ent.detailspanel.category, date = ent.detailspanel.date, filecount = ent.detailspanel.filecount, filesize = ent.detailspanel.filesize, tags = ent.detailspanel.tags }; eh_entrysave entsavedata = new eh_entrysave { dl_location = ent.dl_location, files = ent.file, title = ent.title, detail_savedata = detpane }; db.eh_save.add(url, entsavedata); } saveasbinary(application.startuppath + "\\" + db.db_name + ".dat", db); } public void loaddatabase(string databasepath) { savedatabaseindex loadeddata = loadasbinary(databasepath) savedatabaseindex; //populate required objects loaded data if (loadeddata != null) { cur_page = loadeddata.current_page; firstentryonload = loadeddata.firstentryonsave; urllist = loadeddata.urllist; parallel.foreach(urllist, enturl => { eh_entrysave ent_save = loadeddata.eh_save[enturl] eh_entrysave; htentry.add(enturl, new eh_entry { detailspanel = new details_pane { rating = ent_save.detail_savedata.rating, uploader = ent_save.detail_savedata.uploader, category = ent_save.detail_savedata.category, date = ent_save.detail_savedata.date, filecount = ent_save.detail_savedata.filecount, filesize = ent_save.detail_savedata.filesize, tags = ent_save.detail_savedata.tags }, dl_location = ent_save.dl_location, title = ent_save.title }); }); } } static object loadasbinary(string fullpath) { string filename = fullpath; if (file.exists(filename)) { try { filestream fs = new filestream(filename, filemode.open, fileaccess.read); binaryformatter bf = new binaryformatter(); var loadeddata = bf.deserialize(fs); return loadeddata; } catch (exception ex) { console.writeline("exception: " + ex.message); } } else if (file.exists(filename + "bak")) filename = filename + "bak"; try { filestream fs = new filestream(filename, filemode.open, fileaccess.read); binaryformatter bf = new binaryformatter(); var loadeddata = bf.deserialize(fs); return loadeddata; } catch (exception ex) { console.writeline("exception: " + ex.message); } } return null; }
edit: clarify looking existing fileformat or db product use instead of using binary serialization.
edit2: question not worded great , ended finding answer problem here which database recommend use c# (.net) application?
Comments
Post a Comment