Creating Large Text File Very Slow...

Question

I have a application that reads in text files, pulls info out and create one or many new text files based on infomation in the original file.  I have done this with VB6 and it flys...now c# appears to be very slow at doing this...

I often deal with fairly large files, I noticed my problem when I came accross a 7+ gig file that after an hour was not done, but it was still processing (this same file was done in VB6 in about a minute or less). Here I am placing the lines I want to write out to a text file in a new string and print the file when completed.  I have tried writing the text file line by line, printing into a stringwriter and than print my text file...and this way...Here is a chunk of the code that is being used for this...

public static string ProcessFile(string sFilePath, string sDataDirectory)

{

string sLine;

//This seems to go very fast....

StreamReader sr = new StreamReader(sFilePath);

sLine = sr.ReadToEnd();

sr.Close();

//I do Some Other Functions Here...Determine The File Type And What I Want To Do With It...

//Call Next Function...

sReturn = GeneralFunctions.Process835(sLine, sFilePath, sDataDirectory);

}

public static string Process835(string sFileString, string sFilePath, string sDataDirectory)

{

//Divide By ISA Segments...

//Check If ANSI File...

if (sFileString.Substring(0, 3) == "ISA")

{

string sBodyStream = "";

string sISAStream = "";

string sIEAStream = "";

string sFileName = "";

string sReturnMsg = "";

string sPayorID = "";

string sClaimType = "";

string sLine = "";

string[] aRetSeg;

bool bEOF = false;

bool bReject = false;

long lCtr = 0;

try

{

string sSubEleSep = Convert.ToString(sFileString.Substring(104, 1));

string sEleSep = Convert.ToString(sFileString.Substring(3, 1));

string sTerm = Convert.ToString(sFileString.Substring(105, 1));

long lCurChar = 0;

while (!bEOF)

{

aRetSeg = GeneralFunctions.GetNextLine(sFileString, ref lCurChar, sSubEleSep, sEleSep, sTerm, ref bEOF, ref sLine);

lCtr = 0;

switch (aRetSeg[0])

{

case "ISA":

sISAStream = "";

//Create IEA Segment...We Need It Ahead Of Time And We Need ISA 13

sIEAStream = "IEA" + sEleSep + "1" + sEleSep + aRetSeg[13] + sTerm;

sISAStream = sLine;

break;

case "IEA":

sReturnMsg = sReturnMsg + Program.CreateNewFile(sFileName, "835", sISAStream + sBodyStream + sIEAStream, ref bReject, sDataDirectory, sPayorID, sClaimType) + " \r\n";

break;

case "":

bEOF = true;

break;

default:

if (aRetSeg[lCtr] == "GS")

{

sFileName = aRetSeg[2];

sBodyStream = "";

}

sBodyStream = sBodyStream + sLine;

break;

}

}

return sReturnMsg;

}

catch (Exception e)

{

return "835 File Exception: " + e.Message + "\r\n";

}

}

else

{

return "File Appears To Not Be An ANSI File.(Process835)..\r\n";

}

}

public static string[] GetNextLine(string sFileString, ref long lCurChar, string sSegSubEleSep, string sSegEleSep, string sSegTerm, ref Boolean bEOF, ref string sLine)

{

long i = 0;

long lHoldCurChar = lCurChar;

string sTempLine = "";

System.Text.Encoding enc = System.Text.Encoding.ASCII;

byte[] myByteArray = enc.GetBytes(sFileString);

for (i = lHoldCurChar; i <= Convert.ToDouble(sFileString.Length); i++)

{

char cChar = Convert.ToChar(myByteArray[i]);

if (cChar == Convert.ToChar(sSegTerm))

{

break;

}

else

{

if (Convert.ToInt64(cChar) != 13 & Convert.ToInt64(cChar) != 10 & Convert.ToInt64(cChar) != 12)

{

sTempLine = sTempLine + Convert.ToString(cChar);

}

}

}

sTempLine = sTempLine + sSegTerm;

sLine = sTempLine;

lCurChar = i + 1;

int k = 0;

int j = 0;

string sCurEle = "";

string[] aRetAry = new string[30];

while (j < Convert.ToInt64(sLine.Length))

{

if (sLine.Substring(j, 1) == sSegEleSep | sLine.Substring(j, 1) == Convert.ToString(sSegTerm))

{

aRetAry[k] = sCurEle;

sCurEle = "";

k++;

}

else

{

if (sLine.Substring(j, 1) != Convert.ToString(sSegTerm))

{

sCurEle = sCurEle + sLine.Substring(j, 1);

}

}

j++;

}

if (lCurChar + 5 >= sFileString.Length)

{

bEOF = true;

}

return aRetAry;

}

//This Function Actually Creates The File...

public static string CreateNewFile(string sTransID, string sFileType, string sFileStream, ref bool bError, string sDataDirectory, string sPayorID, string sClaimType)

{

string sPayorDir = sDataDirectory + @"ClaimsTransmissions\";

if (sClaimType != "")

{

sPayorDir = sPayorDir + sClaimType + @"\" + sPayorID + @"\";

}

else

{

sPayorDir = sPayorDir + @"_Files\";

}

if (sFileType == "835")

{

sPayorDir = sPayorDir + @"835\Working\";

}

try

{

try

{

if (!Directory.Exists(sPayorDir))

{

Directory.CreateDirectory(sPayorDir);

}

}

catch

{

bError = true;

return "Error Occured Checking If Folder Path Exists And Create Path If Not...: '" + sDataDirectory + @"ClaimsTransmissions\_Files\" + sFileType + @"\";

}

try

{

//Write File...

StreamWriter writer = File.CreateText(sPayorDir + sTransID + "." + sFileType);

writer.Write(sFileStream);

writer.Close();

bError = false;

return "File Created: '" + sPayorDir + sTransID + "." + sFileType + "'";

}

catch (Exception ex)

{

bError = true;

return "Error Occured Writing New File...: '" + sPayorDir + sTransID + "." + sFileType + "' \r\n" + "Exception Message: '" + ex.Message + "'";

}

}

catch (Exception e)

{

bError = true;

return "Error Occured...Unkown Problem...Exception Message: \r\n" + e.Message;

}

}

 

Thanks for all your help in advance..

Chad