   C++ vs. C# in Chinese-English dictionary test
Edit:Update: even faster version, no boost, no external library: http://www.gamedev.net/community/forums/topic.asp?topic_id=567483&whichpage=1� (link to post in this thread)A recent thread post (http://www.gamedev.net/community/forums/topic.asp?topic_id=567315&whichpage=1�) intrigued me.It links to this page: http://www.codinghorror.com/blog/2005/05/on-managed-code-performance-again.html, which, in turn, links to this page: http://blogs.msdn.com/ricom/archive/2005/05/10/416151.aspx.Something didn't seem right:Quote:Write his own file/io stuffWrite his own string classWrite his own allocatorWrite his own international mappingWhat?I went to this page: http://www.mdbg.net/chindict/chindict.php?page=cedict and got the UTF-8 version of the dictionary.Then I used this C# code:using System;using System.IO;using System.Text;using System.Collections;namespace Program{class Test{[System.Runtime.InteropServices.DllImport("Kernel32.dll")]private static extern bool QueryPerformanceCounter(out long lpPerformanceCount);[System.Runtime.InteropServices.DllImport("Kernel32.dll")]private static extern bool QueryPerformanceFrequency(out long lpFrequency);static void Main(string[] args){long startTime, endTime, freq;QueryPerformanceFrequency(out freq);QueryPerformanceCounter(out startTime);Dictionary dict = new Dictionary();QueryPerformanceCounter(out endTime);Console.WriteLine("length: {0}", dict.Length());Console.WriteLine("frequency: {0}", freq);Console.WriteLine("time: {0}s", (endTime - startTime) / (double)freq);}class DictionaryEntry{private string trad;private string pinyin;private string english;static public DictionaryEntry Parse(string line){DictionaryEntry de = new DictionaryEntry();int start = 0;int end = line.IndexOf(' ', start);if (end == -1) return null;de.trad = line.Substring(start, end - start);start = line.IndexOf('[', end);if (start == -1) return null;end = line.IndexOf(']', ++start);if (end == -1) return null;de.pinyin = line.Substring(start, end - start);start = line.IndexOf('/', end);if (start == -1) return null;start++;end = line.LastIndexOf('/');if (end == -1) return null;if (end <= start) return null;de.english = line.Substring(start, end - start);return de;}public override string ToString(){return english;}};class Dictionary{ArrayList dict;public Dictionary(){StreamReader src = new StreamReader("cedict_ts.u8",System.Text.Encoding.UTF8);string s;DictionaryEntry de;dict = new ArrayList();while ((s = src.ReadLine()) != null){if (s.Length > 0 && s[0] != '#'){if (null != (de = DictionaryEntry.Parse(s))){dict.Add(de);}}}}public int Length() { return dict.Count; }};}}Notice a small change from his original:StreamReader src = new StreamReader("cedict_ts.u8", System.Text.Encoding.UTF8);I changed it to the UTF-8 file instead of the original format he used, which was really weird. (theres also a small change to the Console.WriteLine() calls so it's easier to read, for me at least)That C# code produced this output on my machine:length: 97587frequency: 2511500000time: 0.265279260601234s(hand-selected best output out of a couple of manual runs; best as in fastest time)Fair enough.  After about 3 hours, this is my final C++ code:#include <fstream>#include <iostream>#include <string>#include <algorithm>#include <cstdlib>#include <boost/ptr_container/ptr_vector.hpp>#include <boost/shared_ptr.hpp>#include "utf8.h"#define WIN32_LEAN_AND_MEAN#define WIN32_EXTRA_LEAN#define NOMINMAX#include <Windows.h>// necessary?typedef boost::shared_ptr<std::wstring> strPtr;bool parseTo(const wchar_t *&p, wchar_t delim, const wchar_t * const end){while (*p != delim && p != end)++p;if (p == end)return false;return true;}bool parseToAndSkip(const wchar_t *&p, wchar_t delim, const wchar_t * const end){if (!parseTo(p, delim, end))return false;if (++p == end)return false;return true;}wchar_t *copyOf(const wchar_t * const begin, const wchar_t * const end){const size_t SIZE = end - begin;wchar_t *ret = new wchar_t[SIZE + 1];std::memcpy(ret, begin, SIZE * sizeof(wchar_t));//std::copy(begin, end, ret);ret[SIZE] = '\0';return ret;}class DictionaryEntry{private:DictionaryEntry(const DictionaryEntry&);DictionaryEntry &operator=(const DictionaryEntry&);public:wchar_t *trad, *pinyin, *english;DictionaryEntry(wchar_t *trad, wchar_t *pinyin, wchar_t *english):trad(trad), pinyin(pinyin), english(english){}~DictionaryEntry(){delete[] trad;delete[] pinyin;delete[] english;}static DictionaryEntry *parse(const wchar_t *line, size_t len){const wchar_t * const end = line + len;const wchar_t *parseEnd = line;if (!parseTo(parseEnd, L' ', end))return NULL;//strPtr trad(new std::wstring(line, parseEnd));wchar_t *trad = copyOf(line, parseEnd);line = parseEnd;if (!parseToAndSkip(line, L'[', end))return NULL;parseEnd = line;if (!parseTo(parseEnd, L']', end))return NULL;//strPtr pinyin(new std::wstring(line, parseEnd));wchar_t *pinyin = copyOf(line, parseEnd);line = parseEnd;if (!parseToAndSkip(line, L'/', end))return NULL;parseEnd = end;--parseEnd;while (*parseEnd != '/' && parseEnd != line)--parseEnd;if (parseEnd == line)return NULL;//strPtr english(new std::wstring(line, parseEnd));wchar_t *english = copyOf(line, parseEnd);//return new DictionaryEntry(trad, pinyin, english);return new DictionaryEntry(trad, pinyin, english);}};class Dictionary{private:boost::ptr_vector<DictionaryEntry> dict;public:Dictionary(){std::ifstream in;in.open("cedict_ts.u8", std::ios::binary);assert(in.good());in.seekg(0, std::ios::end);const size_t FILE_SIZE = in.tellg();in.seekg(0, std::ios::beg);char *buf = new char[FILE_SIZE];char *bufEnd = buf + FILE_SIZE;in.read(buf, FILE_SIZE);wchar_t line[4096];size_t lineLen = 0;char *lineCur = buf;while (lineCur < bufEnd){wchar_t c = utf8::next(lineCur, bufEnd);if (c == '\n' || lineCur == bufEnd){// process lineif (lineLen > 0){if (line[0] != '#') // comment{DictionaryEntry *de = DictionaryEntry::parse(line, lineLen);if (de)dict.push_back(de);}lineLen = 0;}}else if (c != 13) // carriage return on windows{line[lineLen++] = c;}}delete buf;}size_t length() const{return dict.size();}};int main(int argc, char *argv[]){LARGE_INTEGER startTime, endTime, freq;QueryPerformanceFrequency(&freq);QueryPerformanceCounter(&startTime);Dictionary dict;QueryPerformanceCounter(&endTime);std::cout << "length: " << dict.length() << "\n";std::cout << "frequency: " << freq.QuadPart << "\n";std::cout << "time: " << (endTime.QuadPart - startTime.QuadPart) / (double)freq.QuadPart << "s\n";}Indeed, I used the following UTF-8 helper library: http://utfcpp.sourceforge.net/. It's a header-only library with 4 small files, and considering that the C++ standard doesn't really have any good unicode stuff (unlike C#), it was fair enough to use this library. (took all of maybe 1 minute to set it up)The above code produced this output on my machine:length: 97587frequency: 2511500000time: 0.233198s(hand-plucked in the same way as the C# output)I spot a 0.03s difference, in C++'s favor.  Fair enough.Before you read on, please confirm my results for yourself.  Also, please check my C++ code and see if I made any logical mistakes compared to the original.  After all, it's lated at night and I'm pretty tired [grin]Anyways, to continue: let me get this right... This guy, a Microsoft professional, had to:Quote:Write his own file/io stuffWrite his own string classWrite his own allocatorWrite his own international mappingWhereas I, a 10th grade high school student, had to:Quote:Download quick UTF-8 library for WindowsUse some plain C style string manipulation instead of wasting memory everywhere with temporary C++ style stringsNow - the difference is that he originally used the big5 version, whereas I used the UTF-8 version.  That's a pretty valid argument... just about as valid as comparing a dirt race car to a road race car on a dirt path.Anyways, yeah, the C++ version is definitely uglier, and took longer to write.  And that's really what C# is for - GUI stuff, business applications that have to deal with various encodings, and so on.In fact, I use C# quite a lot myself. I definitely prefer it over C++ for a lot of things.But come on now, nobody likes exaggerations.Discuss [smile](Disclaimer: if I'm blatantly wrong somewhere, please excuse this whole thread, I'm tired - but I do think the thread is correct [smile])[Edited by - nullsquared on April 7, 2010 3:04:18 PM]
  You're not converting the data from whatever format it is into proper unicode. Raymond Chen's code (due to the lack of a unicode dictionary input) had to go off and convert all the symbols into proper unicode code-points.   ;  You know, I just spent an hour trying to refute your rather snarky post, but your results are pretty spot on and not terribly unsurprising. C# and C++ perform pretty similarly for most things, except C# is quicker to write and results in more readable code.One thing that is a difference between your version and the original is the use of boost::ptr_vector instead of std::vector. The original doesn't disable the microsoft bounds checking, which might be a moot point if they were using gcc, but is pretty significant if they were using msvs. Doing my own testing, leaving that enabled does produce the sort of performance disparity shown in the article. *shrug*   ;  I get the impression that your C++ code could be alot shorter if you'd just use the string functions provided by the SC++L. It might also be faster for short strings, the string classes have some magic included for that case. And it *might* be faster if you use a custom allocator and/or simply a pre-sized std::vector storing the entries without pointers.On a second look, you can do some real nasty trickery by making all dictionary entries only referring to the characters in the original file, instead of copying them into freshly allocated wchar arrays. That's the difference why C++ is still used, I think: you *can* do such magic. And it will reach performance regions where no managed language can follow. It's just that such magic gets increasingly unimportant. After all, a quarter of a second is still fast enough.Can't do any actual checks here, though, because I'm at work with no C# compiler available on my machine.   ;Quote:Original post by bobofjoeYou're not converting the data from whatever format it is into proper unicode. Raymond Chen's code (due to the lack of a unicode dictionary input) had to go off and convert all the symbols into proper unicode code-points.Well all I'm doing is using UTF-8 (via UTF8-CPP in C++ and just the standard stuff in C#). I didn't want to mess with big5 since I'm not familiar with it. In fact, if anything, the big5 version might actually be better in C++'s favor because big5 is a double-byte character set, whereas UTF-8 is variable-length bytes.Quote:Original post by SchrompfI get the impression that your C++ code could be alot shorter if you'd just use the string functions provided by the SC++L.Well, yeah, but then it'd also be slower. I tried to make it so that most things are done in-place and with as little allocation as possible. Even so, my C++ code is still rather readable, and only took a couple of hours to write, whereas Raymond Chen apparent had to:Quote:Write his own file/io stuffWrite his own string classWrite his own allocatorWrite his own international mapping... over the period of a week I reckon.  That's why I'm so surprised.Quote:Original post by TelastynYou know, I just spent an hour trying to refute your rather snarky post, but your results are pretty spot on and not terribly unsurprising. C# and C++ perform pretty similarly for most things, except C# is quicker to write and results in more readable code.Thanks for checking it over, and I completely agree. C# *is* quicker. For example, I wrote my little competition judge in C# in the course of several revisions, for a total of maybe 5 hours. Looking at all the features it supports, all with standard C# code, there's no way I could've done the same thing in the same time with standard C++.Quote:One thing that is a difference between your version and the original is the use of boost::ptr_vector instead of std::vector.Well, this guy went really h4rdc0re on those optimizations, I'd assume he'd be able to disable the debugging/safety checks. That plus I didn't want any copies being made so ptr_vector was my immediate choice.Now: his "optimized" C# version indeed runs faster than my C++ version. But my point is still valid, as he compared his C++ optimizations to the original C# code, and *then* wrote a C# version which beat even the optimized C++ version.   ;  Just to demonstrate that c++ can be as readable as c#:#include <QtCore/QCoreApplication>#include <QFile>#include <QTextStream>#include <QList>#include <QStringList>#include <QRegExp>#include <stdexcept>#include <QDebug>class DictionaryEntry{    QString trad;    QString pinyin;    QString english;public:    DictionaryEntry(QString line) throw(std::runtime_error)    {QRegExp regex("(.+)\\s(.+)\\s+\\[(.+)\\]\\s+/(.+)/$");if(0!=regex.indexIn(line)) throw std::runtime_error("Could not parse an entry");QStringList list = regex.capturedTexts();trad = list.at(1);pinyin = list.at(3);english = list.at(4);    }};class Dictionary{QList<DictionaryEntry> dict;public:Dictionary(){    QFile dicFile("cedict_ts.u8");    dicFile.open(QFile::ReadOnly);    QTextStream src(&dicFile);    src.setCodec("UTF-8");    QString line;    while (!(line = src.readLine()).isNull())    {if(line[0]==QChar('#')) continue;try{    DictionaryEntry de(line);    dict.append(de);} catch(std::exception& e){    qWarning() << e.what() ;}    }}int Length() { return dict.length(); }};int main(int argc, char *argv[]){    QCoreApplication a(argc, argv);    Dictionary dict;    qDebug() << dict.Length();    return 0;}Using Qt here, although I agree that c# is a nice language. Speed comparisons like this are meaningless in my option, if you want to compare compilers, you need to give them same code to optimize (also a bit more challenging code that doesn't just consist of calling some function from the standard library which is probably optimized itself. ). If you want to compare the languages then go for shortest code, or best readable code or something like that.Also for scientific computations that take a lot of time people usually use the Intel compiler which is much better that MS and GNU compilers. As far as I know there are also no equivalents for libraries like Intel Math Kernel library in c#. What I want to say is when performance matter, people are willing to write ugly code (even assembly) and c# has no chance to beat c++.But for desktop applications, tools and games, c# is fine. (If you don't care too much about platforms other than Windows)[Edited by - Kambiz on April 7, 2010 7:56:54 AM];  Pssh, regexes are cheating. C# code looks a lot better if you go that route too:using System;using System.Collections.Generic;using System.Linq;using System.Text;using System.Text.RegularExpressions;namespace Csharp_chindict {    class Program {   [System.Runtime.InteropServices.DllImport("Kernel32.dll")]   private static extern bool QueryPerformanceCounter(out long lpPerformanceCount);   [System.Runtime.InteropServices.DllImport("Kernel32.dll")]   private static extern bool QueryPerformanceFrequency(out long lpFrequency);   static void Main(string[] args) {  long start, end, freq;  Regex lineParser = new Regex(@"(.*) .*\[(.*)].*/(.*)/", RegexOptions.Compiled);  QueryPerformanceFrequency(out freq);  QueryPerformanceCounter(out start);  List<dynamic> dict = new List<dynamic>();  foreach (string line in System.IO.File.ReadAllLines(@"c:\src\Csharp-chindict\Csharp-chindict\cedict_ts.u8")) { if (line.Length != 0) {if (line[0] != '#') {    Match result = lineParser.Match(line);    if (result.Success) {   dict.Add(new { traditional = result.Groups[1].Value, pinyin = result.Groups[2].Value, english = result.Groups[3].Value });    }} }  }  QueryPerformanceCounter(out end);  Console.WriteLine("Count: {0}", dict.Count);  Console.WriteLine("Ticks: {0}", (end-start)/(double)freq);   }    }}Though this version is decidedly less performant (though that might be due to anonymous type shenanigans...).   ;Quote:Original post by nullsquaredIndeed, I used the following UTF-8 helper library: http://utfcpp.sourceforge.net/.Did you get the library verified by the lawyer to ensure it is compatible with your project's and company release policy? Did he verify that the implementation does not violate any patents? Did you verify that the author of the library, or any third parties associated with the library are not residents of a country that is banned from doing trade in any of the markets where you wish to ship? Has all of the above been documented and filed? Has the code in the library been verified as original work, or has author signed a "work-for-hire" or other form of transfer of ownership to your company?Did you submit the library to your build master for integration with the build system? Have the integration and other tests been written?Did you coordinate the introduction of new third-party library with the project manager for inclusion into the project life-cycle? Has risk analysis with regard to third-party library lifecycle been integrated into your product's life cycle?All of the above are issues that will need to be cleared in any reasonably-sized company. This 1 minute download just caused 50 hour workload for your company across multiple departments.Imagine that library happens to contain something that violates some patent (perhaps it uses a linked list), and it gets integrated into your flagship product. Suddenly, you have IBM suing your for hundreds of millions of dollars.This is why open source matters only in some rare cases, the rest of the code lying around source sites simply doesn't exist. It's too expensive.As Microsoft once said, Linux is only free if your time is worthless. Most people do not understand that and laugh at Microsoft. And for a startup with no revenue it's true, FOSS is great. But as soon as money is involved, things get very complicated.The benefit of built-in libraries, or those provided by software vendors is that they are safe from most of the above. Hence, writing something from scratch, despite potential bugs, is considerably better choice in such setting.   ;  To the OP:Raymond Chen's final version took 70ms to execute.  Yours took 230 ms to execute.  So his still beats yours by a factor of 3.  Not saying your code sucks or anything by any means, but just saying you're doing a little injustice to the amount of work he put in.  Also as mentioned having an already optimized UTF8 library to help you out certainly helped reduce your development time.   ;Quote:Original post by KambizJust to demonstrate that c++ can be as readable as c#:Readability wasn't the point. Speed was.Quote:Original post by Antheus[completely off-topic blabbing]All of the above are issues that will need to be cleared in any reasonably-sized company. This 1 minute download just caused 50 hour workload for your company across multiple departments.Yeah, great. No offence, Antheus, but none of what you said applies here.Quote:Raymond Chen is running a series of articles about how to build and optimize the startup time of a Chinese/English dictionary.He's running a series of articles on how to build and optimize the startup time of a Chinese/English dictionary.  He's not developing a propriety Chinese/English dictionary for some major company.  Notice the words "build and optimize."  This is an optimization programming problem, not an actual tool to be used by a large company.  In fact, notice the title: "Performance Quiz #6" - a performance quiz.Please do not try to thwart me with a bunch of legal bullshit that doesn't apply to the situation.  But - if it bothers you that much - I actually checked the library.  I took the liberty of taking 5 minutes to go through the 4 files of the source.  Here are my observations:1) open source license that permits me to "[...] use, reproduce, display, distribute,execute, and transmit the Software, and to prepare derivative works of theSoftware, and to permit third-parties to whom the Software is furnished todo so [...]"2) the library only uses the SC++L and has no other dependenciesQuote:Original post by cache_hitTo the OP:Raymond Chen's final version took 70ms to execute.  Yours took 230 ms to execute.  So his still beats yours by a factor of 3.  Just to be clear, which versions are we talking about?  Obviously we have different machines, so you can't just compare our two numbers - the various tests need to be run on the same machine.  Or did you run them for yourself and report these numbers?  All I'm saying is that his first managed version ran at the same speed as his version 5 unmanaged optimized version, which, apparently, took a week to finish; and, thus, I'm showing that his first managed version runs slower than my first unmanaged optimized version, which took 3 hours to finish.  That is *all* I'm saying.   