DocumentServer/ServerComponents/FileConverterService2/FileConverter.cs

1004 lines
52 KiB
C#
Raw Normal View History

2015-04-28 16:22:25 +00:00
/*
* (c) Copyright Ascensio System SIA 2010-2015
*
* This program is a free software product. You can redistribute it and/or
* modify it under the terms of the GNU Affero General Public License (AGPL)
* version 3 as published by the Free Software Foundation. In accordance with
* Section 7(a) of the GNU AGPL its Section 15 shall be amended to the effect
* that Ascensio System SIA expressly excludes the warranty of non-infringement
* of any third-party rights.
*
* This program is distributed WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. For
* details, see the GNU AGPL at: http://www.gnu.org/licenses/agpl-3.0.html
*
* You can contact Ascensio System SIA at Lubanas st. 125a-25, Riga, Latvia,
* EU, LV-1021.
*
* The interactive user interfaces in modified source and object code versions
* of the Program must display Appropriate Legal Notices, as required under
* Section 5 of the GNU AGPL version 3.
*
* Pursuant to Section 7(b) of the License you must retain the original Product
* logo when distributing the program. Pursuant to Section 7(e) we decline to
* grant you any rights under trademark law for use of our trademarks.
*
* All the Product's GUI elements, including illustrations and icon sets, as
* well as technical writing content are licensed under the terms of the
* Creative Commons Attribution-ShareAlike 4.0 International. See the License
* terms at http://creativecommons.org/licenses/by-sa/4.0/legalcode
*
*/
using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Configuration;
using System.Threading;
using System.IO;
using System.Net;
using System.Xml;
using System.Text;
using System.Web;
using FileConverterUtils2;
using log4net;
using log4net.Config;
namespace FileConverterService2
{
public static class StringParser
{
public static string[] ParseArguments(string commandLine)
{
char[] parmChars = commandLine.ToCharArray();
bool inQuote = false;
for (int index = 0; index < parmChars.Length; index++)
{
if (parmChars[index] == '"')
inQuote = !inQuote;
if (!inQuote && parmChars[index] == ' ')
parmChars[index] = '\n';
}
char[] oDelimiters = new char[] { '\n' };
return (new string(parmChars)).Split(oDelimiters, StringSplitOptions.RemoveEmptyEntries);
}
}
public class FileConverter
{
private static readonly ILog _log = LogManager.GetLogger(typeof(FileConverter));
private class IddleProcess
{
public System.Diagnostics.Process m_oProcess;
public string m_sDirToDelete;
public string m_sDirSource;
public string m_sDirResult;
public string m_sFileTo;
public string m_sChangesAuthor = null;
public TaskQueueData m_oTaskQueueData;
}
private class TransportClass
{
public TaskQueueData m_oTaskQueueData;
public AsyncWebRequestOperation m_oAsyncWebRequestOperation;
public IAsyncResult m_oAsyncWebRequestOperationResult;
}
#if! OPEN_SOURCE
private static ASCGraphics.CASCWinFonts piWinFonts = null;
#else
private static OfficeCore.CWinFontsClass piWinFonts = null;
#endif
private int m_nMaxRunThreads;
private readonly ManualResetEvent _stopEvt = new ManualResetEvent(false);
private List<Thread> m_aRunThreads = new List<Thread>();
private Thread m_oIddlePriority = null;
private List<IddleProcess> m_arrIndentFiles = new List<IddleProcess>();
private Thread m_oGCThread = null;
private object m_oKeyToPercentLock = new object();
private object m_oKeyToPercentValidateLock = new object();
private Dictionary<string, int> m_mapKeyToPercentValidate = new Dictionary<string, int>();
private Dictionary<string, int> m_mapKeyToPercent = new Dictionary<string, int>();
private Timer m_oPercentTimer = null;
private static int m_nRunThreadCount = 0;
public FileConverter(string serviceName)
{
m_nMaxRunThreads = (int)(Environment.ProcessorCount * double.Parse(ConfigurationSettings.AppSettings["maxprocesscount"], new System.Globalization.CultureInfo(0x409)));
if (0 >= m_nMaxRunThreads)
m_nMaxRunThreads = 1;
}
public void Start()
{
if (null == piWinFonts)
{
bool bIsUseWinFonts = bool.Parse(ConfigurationSettings.AppSettings["fileconverterservice.usewinfonts"] ?? "true");
if (bIsUseWinFonts)
{
_log.Info("Create WinFonts.");
#if! OPEN_SOURCE
piWinFonts = new ASCGraphics.CASCWinFonts();
#else
piWinFonts = new OfficeCore.CWinFontsClass();
string sFontDir = ConfigurationSettings.AppSettings["utils.common.fontdir"] ?? "";
if(null != sFontDir && !string.IsNullOrEmpty(sFontDir))
sFontDir = Path.GetFullPath(Environment.ExpandEnvironmentVariables(sFontDir));
piWinFonts.Init(sFontDir, true, true);
#endif
}
}
Stop();
_stopEvt.Reset();
for (int i = 0; i < m_nMaxRunThreads; i++)
{
var runThread = new Thread(Run);
_log.InfoFormat("Start convertation thread {0} of {1}.", i + 1, m_nMaxRunThreads);
runThread.Start();
m_aRunThreads.Add(runThread);
}
m_oGCThread = new Thread(GCThread);
_log.Info("Start garbage collector thread.");
m_oGCThread.Start();
}
public void Stop()
{
_stopEvt.Set();
foreach (var runThread in m_aRunThreads)
{
try
{
if (!runThread.Join(TimeSpan.FromSeconds(30)))
{
runThread.Abort();
}
}
catch(Exception e)
{
_log.Error("Exception catched while thread stoping.", e);
}
}
m_aRunThreads.Clear();
m_arrIndentFiles.Clear();
if (null != m_oGCThread)
{
m_oGCThread.Abort();
}
}
private void Run()
{
FileConverterUtils2.CTaskQueue oTaskQueue = new FileConverterUtils2.CTaskQueue();
while (!_stopEvt.WaitOne(0))
{
FileConverterUtils2.TaskQueueData oTaskQueueData = oTaskQueue.GetTask();
if (null != oTaskQueueData)
{
DateTime oGetTaskTime = DateTime.UtcNow;
_log.DebugFormat("Start Task(id={0}), Time={1}", oTaskQueueData.m_sKey, oGetTaskTime);
Interlocked.Increment(ref m_nRunThreadCount);
ErrorTypes eError = FileConverterUtils2.ErrorTypes.NoError;
bool bNeedParam = false;
ITaskResultInterface oTaskResult = TaskResult.NewTaskResult();
Storage oFileStore = new Storage();
string sDirToDelete = Utils.GetTempDirectory();
IddleProcess oNewIddleProcess = new IddleProcess();
oNewIddleProcess.m_sDirToDelete = sDirToDelete;
oNewIddleProcess.m_oTaskQueueData = oTaskQueueData;
try
{
string sTempDir = Path.Combine(sDirToDelete, oTaskQueueData.m_sKey);
Directory.CreateDirectory(sTempDir);
string sDirSource = Path.Combine(sTempDir, "source");
Directory.CreateDirectory(sDirSource);
string sDirResult = Path.Combine(sTempDir, "result");
Directory.CreateDirectory(sDirResult);
oNewIddleProcess.m_sDirSource = sDirSource;
oNewIddleProcess.m_sDirResult = sDirResult;
int nFormatFrom = FileFormats.AVS_OFFICESTUDIO_FILE_UNKNOWN;
string sFileFrom = "";
string sFileTo = Path.Combine(sDirResult, oTaskQueueData.m_sToFile);
if (false == string.IsNullOrEmpty(oTaskQueueData.m_sFromUrl))
{
sFileFrom = Path.Combine(sDirSource, oTaskQueueData.m_sKey + "." + oTaskQueueData.m_sFromFormat);
eError = DownloadFile(oTaskQueueData.m_sFromUrl, sFileFrom);
_log.DebugFormat("DownloadFile complete(id={0})", oTaskQueueData.m_sKey);
if (FileConverterUtils2.ErrorTypes.NoError == eError)
{
nFormatFrom = FileConverterUtils2.FormatChecker.GetFileFormat(sFileFrom);
switch(nFormatFrom)
{
case FileFormats.AVS_OFFICESTUDIO_FILE_DOCUMENT_HTML:
{
if (".html" != Path.GetExtension(sFileFrom))
{
string sNewFileFrom = Path.ChangeExtension(sFileFrom, ".html");
File.Move(sFileFrom, sNewFileFrom);
sFileFrom = sNewFileFrom;
oTaskQueueData.m_sFromFormat = "html";
}
}
break;
case FileFormats.AVS_OFFICESTUDIO_FILE_OTHER_MS_OFFCRYPTO:
eError = ErrorTypes.ConvertMS_OFFCRYPTO;
break;
case FileFormats.AVS_OFFICESTUDIO_FILE_UNKNOWN:
eError = ErrorTypes.ConvertUnknownFormat;
break;
case FileFormats.AVS_OFFICESTUDIO_FILE_DOCUMENT_TXT:
{
if (false == oTaskQueueData.m_nCsvTxtEncoding.HasValue)
{
System.Text.Encoding oEncoding = GetEncoding(sFileFrom);
if (null == oEncoding)
{
string sFileFromName = Path.GetFileName(sFileFrom);
string sJson = Utils.GetSerializedEncodingProperty(sFileFromName, null, null);
int nReadWriteBytes;
byte[] aSettingsPreamble = System.Text.Encoding.UTF8.GetPreamble();
byte[] aSettingsContent = System.Text.Encoding.UTF8.GetBytes(sJson);
byte[] aSettingsData = new byte[aSettingsPreamble.Length + aSettingsContent.Length];
aSettingsPreamble.CopyTo(aSettingsData, 0);
aSettingsContent.CopyTo(aSettingsData, aSettingsPreamble.Length);
using (MemoryStream ms = new MemoryStream(aSettingsData))
oFileStore.WriteFile(Path.Combine(oTaskQueueData.m_sKey, "settings.json"), ms, out nReadWriteBytes);
using (FileStream fs = new FileStream(sFileFrom, FileMode.Open))
oFileStore.WriteFile(Path.Combine(oTaskQueueData.m_sKey, sFileFromName), fs, out nReadWriteBytes);
bNeedParam = true;
}
else
{
oTaskQueueData.m_nCsvTxtEncoding = oEncoding.CodePage;
}
}
}
break;
case FileFormats.AVS_OFFICESTUDIO_FILE_SPREADSHEET_CSV:
{
if (false == oTaskQueueData.m_nCsvTxtEncoding.HasValue ||
false == oTaskQueueData.m_nCsvDelimiter.HasValue)
{
string sFileFromName = Path.GetFileName(sFileFrom);
int? encoding;
int? delimiter;
GetEncodingAndDelimeter(sFileFrom, out encoding, out delimiter);
string sJson = Utils.GetSerializedEncodingProperty(sFileFromName, encoding, delimiter);
int nReadWriteBytes;
byte[] aSettingsPreamble = System.Text.Encoding.UTF8.GetPreamble();
byte[] aSettingsContent = System.Text.Encoding.UTF8.GetBytes(sJson);
byte[] aSettingsData = new byte[aSettingsPreamble.Length + aSettingsContent.Length];
aSettingsPreamble.CopyTo(aSettingsData, 0);
aSettingsContent.CopyTo(aSettingsData, aSettingsPreamble.Length);
using (MemoryStream ms = new MemoryStream(aSettingsData))
oFileStore.WriteFile(Path.Combine(oTaskQueueData.m_sKey, "settings.json"), ms, out nReadWriteBytes);
using (FileStream fs = new FileStream(sFileFrom, FileMode.Open))
oFileStore.WriteFile(Path.Combine(oTaskQueueData.m_sKey, sFileFromName), fs, out nReadWriteBytes);
bNeedParam = true;
}
}
break;
default:
break;
}
}
}
else if (false == string.IsNullOrEmpty(oTaskQueueData.m_sFromKey))
{
StorageTreeNode oStorageTreeNode = oFileStore.GetTreeNode(oTaskQueueData.m_sFromKey);
eError = DownloadStorageTreeNodeToFilesystem(sDirSource, oTaskQueueData.m_sFromKey, "", oStorageTreeNode, oFileStore);
_log.DebugFormat("DownloadStorageTreeNodeToFilesystem complete(id={0})", oTaskQueueData.m_sKey);
if (oTaskQueueData.m_bFromOrigin.HasValue && true == oTaskQueueData.m_bFromOrigin.Value)
{
sFileFrom = Path.Combine(sDirSource, "origin");
nFormatFrom = FileConverterUtils2.FormatChecker.GetFileFormat(sFileFrom);
}
else if (oTaskQueueData.m_bFromSettings.HasValue && true == oTaskQueueData.m_bFromSettings.Value)
{
string sSettings = Path.Combine(sDirSource, "settings.json");
string sFileFromName;
int? codepage;
int? delimiter;
Utils.GetDeserializedEncodingProperty(File.ReadAllText(sSettings, System.Text.Encoding.UTF8), out sFileFromName, out codepage, out delimiter);
if (null != sFileFromName)
{
sFileFrom = Path.Combine(Path.GetDirectoryName(sSettings), sFileFromName);
nFormatFrom = FileConverterUtils2.FormatChecker.GetFileFormat(sFileFrom);
}
else
eError = ErrorTypes.Unknown;
}
else
{
oStorageTreeNode = oFileStore.GetTreeNode(oTaskQueueData.m_sKey);
eError = DownloadStorageTreeNodeToFilesystem(sDirSource, oTaskQueueData.m_sKey, "", oStorageTreeNode, oFileStore);
if (ErrorTypes.NoError == eError)
{
sFileFrom = Path.Combine(sDirSource, "Editor.bin");
if (FileFormats.AVS_OFFICESTUDIO_FILE_CROSSPLATFORM_PDF == oTaskQueueData.m_nToFormat)
{
string sFileFromWE = Path.GetFileNameWithoutExtension(sFileFrom);
string[] aFiles = Directory.GetFiles(Path.GetDirectoryName(sFileFrom));
Array.Sort<string>(aFiles, Utils.CompareStringByLength);
bool bEmptyFile = true;
string sTempFile = Path.ChangeExtension(sFileFrom, ".tmp");
using (BinaryWriter writer = new BinaryWriter(File.Open(sTempFile, FileMode.Create)))
{
bool bFirst = true;
for (int i = 0, length = aFiles.Length; i < length; ++i)
{
string sCurFilename = aFiles[i];
string sCurFileFromWE = Path.GetFileNameWithoutExtension(sCurFilename);
if (sCurFileFromWE != sFileFromWE && 0 == sCurFileFromWE.IndexOf(sFileFromWE))
{
if (bFirst)
bFirst = false;
else
{
writer.Write('\n');
}
writer.Write(File.ReadAllBytes(sCurFilename));
bEmptyFile = false;
}
}
}
if (false == bEmptyFile)
{
File.Delete(sFileFrom);
File.Move(sTempFile, sFileFrom);
}
File.Delete(sTempFile);
}
else
nFormatFrom = FileConverterUtils2.FormatChecker.GetFileFormat(sFileFrom);
}
}
if (oTaskQueueData.m_bFromChanges.HasValue && oTaskQueueData.m_bFromChanges.Value)
{
List<DocsChange> aChanges;
(new DocsChanges()).GetChanges(oTaskQueueData.m_sFromKey, out aChanges);
string sChangesDir = Path.Combine(sDirSource, "changes");
Directory.CreateDirectory(sChangesDir);
string sChangesAuthor = null;
int nIndexFile = 0;
StreamWriter oStreamWriter = null;
try
{
for (int i = 0, length = aChanges.Count; i < length; ++i)
{
if (null == sChangesAuthor || sChangesAuthor != aChanges[i].userid)
{
if (null != sChangesAuthor)
{
oStreamWriter.Write("]");
oStreamWriter.Dispose();
oStreamWriter = null;
}
sChangesAuthor = aChanges[i].userid;
oStreamWriter = new StreamWriter(Path.Combine(sChangesDir, "changes" + (nIndexFile++) + ".json"));
oStreamWriter.Write("[");
oStreamWriter.Write(aChanges[i].data);
}
else
{
oStreamWriter.Write(",");
oStreamWriter.Write(aChanges[i].data);
}
}
oNewIddleProcess.m_sChangesAuthor = sChangesAuthor;
if (null != oStreamWriter)
{
oStreamWriter.Write("]");
oStreamWriter.Dispose();
oStreamWriter = null;
}
}
catch
{
}
finally
{
if (null != oStreamWriter)
oStreamWriter.Dispose();
}
}
}
else
eError = ErrorTypes.Unknown;
int nToFormat = oTaskQueueData.m_nToFormat;
if (FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS == nToFormat)
{
if (FileFormats.AVS_OFFICESTUDIO_FILE_TEAMLAB_XLSY == nFormatFrom || 0 != (FileFormats.AVS_OFFICESTUDIO_FILE_SPREADSHEET & nFormatFrom))
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET;
else if (FileFormats.AVS_OFFICESTUDIO_FILE_TEAMLAB_PPTY == nFormatFrom || 0 != (FileFormats.AVS_OFFICESTUDIO_FILE_PRESENTATION & nFormatFrom))
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION;
else
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS_WORD;
}
else if (FileFormats.AVS_OFFICESTUDIO_FILE_OTHER_TEAMLAB_INNER == nToFormat)
{
if (FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS_SPREADSHEET == nFormatFrom || FileFormats.AVS_OFFICESTUDIO_FILE_TEAMLAB_XLSY == nFormatFrom || 0 != (FileFormats.AVS_OFFICESTUDIO_FILE_SPREADSHEET & nFormatFrom))
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_SPREADSHEET_XLSX;
else if (FileFormats.AVS_OFFICESTUDIO_FILE_CANVAS_PRESENTATION == nFormatFrom || FileFormats.AVS_OFFICESTUDIO_FILE_TEAMLAB_PPTY == nFormatFrom || 0 != (FileFormats.AVS_OFFICESTUDIO_FILE_PRESENTATION & nFormatFrom))
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_PRESENTATION_PPTX;
else
nToFormat = FileFormats.AVS_OFFICESTUDIO_FILE_DOCUMENT_DOCX;
sFileTo = Path.ChangeExtension(sFileTo, "." + FileFormats.ToString(nToFormat));
}
oNewIddleProcess.m_sFileTo = sFileTo;
if (ErrorTypes.NoError == eError && false == bNeedParam)
{
TimeSpan oWaitTimeout = oTaskQueueData.VisibilityTimeout -
(DateTime.UtcNow - oGetTaskTime);
int nWaitTimeout = Convert.ToInt32(oWaitTimeout.TotalMilliseconds * 0.95);
TaskResultDataToUpdate oTaskResultData = new TaskResultDataToUpdate();
oTaskResultData.eStatus = FileStatus.Convert;
oTaskResultData.nStatusInfo = 0;
oTaskResult.Update(oTaskQueueData.m_sKey, oTaskResultData);
TaskQueueDataConvert oTaskQueueDataConvert = new TaskQueueDataConvert(oTaskQueueData.m_sKey, sFileFrom, nFormatFrom, sFileTo, nToFormat);
if (oTaskQueueData.m_nCsvTxtEncoding.HasValue)
oTaskQueueDataConvert.m_nCsvTxtEncoding = oTaskQueueData.m_nCsvTxtEncoding.Value;
if (oTaskQueueData.m_nCsvDelimiter.HasValue)
oTaskQueueDataConvert.m_nCsvDelimiter = oTaskQueueData.m_nCsvDelimiter;
if (oTaskQueueData.m_bPaid.HasValue)
oTaskQueueDataConvert.m_bPaid = oTaskQueueData.m_bPaid.Value;
if (oTaskQueueData.m_bEmbeddedFonts.HasValue)
oTaskQueueDataConvert.m_bEmbeddedFonts = oTaskQueueData.m_bEmbeddedFonts.Value;
if (oTaskQueueData.m_bFromChanges.HasValue)
oTaskQueueDataConvert.m_bFromChanges = oTaskQueueData.m_bFromChanges.Value;
#if! OPEN_SOURCE
string sFontDir = ConfigurationSettings.AppSettings["utils.common.fontdir"];
#else
string sFontDir = "";
#endif
if (null != sFontDir && !string.IsNullOrEmpty(sFontDir))
oTaskQueueDataConvert.m_sFontDir = Path.GetFullPath(Environment.ExpandEnvironmentVariables(sFontDir));
string sThemeDir = ConfigurationSettings.AppSettings["fileconverterservice.converter.presentationthemesdir"];
if (null != sThemeDir && !string.IsNullOrEmpty(sThemeDir))
oTaskQueueDataConvert.m_sThemeDir = Path.GetFullPath(Path.Combine(Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location), sThemeDir));
lock (m_oKeyToPercentValidateLock)
{
m_mapKeyToPercentValidate[oTaskQueueData.m_sKey] = 1;
}
string sConvertApp = ConfigurationSettings.AppSettings["fileconverterservice.converter.filepath"] ?? "FileConverter2.exe";
string sConvertArgs = ConfigurationSettings.AppSettings["fileconverterservice.converter.args"] ?? "";
System.Diagnostics.Process convertProc = new System.Diagnostics.Process();
oNewIddleProcess.m_oProcess = convertProc;
convertProc.StartInfo.FileName = sConvertApp;
string sConfigPath = Path.Combine(sTempDir, "params.xml");
File.WriteAllText(sConfigPath, TaskQueueDataConvert.SerializeToXml(oTaskQueueDataConvert), System.Text.Encoding.UTF8);
convertProc.StartInfo.Arguments = sConvertArgs + " " + EscapeCommandLineArguments(sConfigPath);
convertProc.StartInfo.CreateNoWindow = true;
convertProc.StartInfo.UseShellExecute = false;
convertProc.OutputDataReceived += new System.Diagnostics.DataReceivedEventHandler(convertProc_OutputDataReceived);
convertProc.StartInfo.RedirectStandardOutput = true;
_log.DebugFormat("Start {0} {1}(id={2})", convertProc.StartInfo.FileName, convertProc.StartInfo.Arguments, oTaskQueueData.m_sKey);
convertProc.Start();
convertProc.BeginOutputReadLine();
bool isProcessExit = convertProc.WaitForExit(nWaitTimeout);
_log.DebugFormat("Stop WaitForExit({0})(id={1})", nWaitTimeout, oTaskQueueData.m_sKey);
if (!isProcessExit)
{
_log.DebugFormat("Kill() process (id={1})", oTaskQueueData.m_sKey);
convertProc.Kill();
}
}
}
catch(Exception e)
{
eError = ErrorTypes.Convert;
_log.Error("Exception catched while convertation thread working.", e);
}
PostProcess(oNewIddleProcess, eError, bNeedParam);
Interlocked.Decrement(ref m_nRunThreadCount);
}
else
{
Thread.Sleep(TimeSpan.FromSeconds(double.Parse(ConfigurationSettings.AppSettings["sleeptimeout"] ?? "60")));
}
}
}
private ErrorTypes DownloadFile(string sUrl, string sFileFrom)
{
ErrorTypes eError = ErrorTypes.NoError;
try
{
AsyncWebRequestOperation oAsyncWebRequestOperation = new AsyncWebRequestOperation(Convert.ToInt64(ConfigurationSettings.AppSettings["maxdownloadbytes"] ?? "10485760"));
byte[] aBuffer;
if (ErrorTypes.NoError == oAsyncWebRequestOperation.Request(sUrl, "GET", null, null, out aBuffer))
{
using(FileStream fs = File.Create(sFileFrom))
{
fs.Write(aBuffer, 0, aBuffer.Length);
}
}
else
{
eError = ErrorTypes.ConvertDownload;
_log.ErrorFormat("DownloadFile() from {0} to {1}", sUrl, sFileFrom);
}
}
catch(Exception ex)
{
eError = ErrorTypes.ConvertDownload;
_log.ErrorFormat("DownloadFile() from {0} to {1}", sUrl, sFileFrom);
_log.Error("Exception:", ex);
}
return eError;
}
private System.Text.Encoding GetEncoding(string sFileFrom)
{
const int nBytesToRead = 3;
byte[] aFirstBytes = new byte[nBytesToRead];
int nBytesReaded = 0;
using (FileStream fs = File.OpenRead(sFileFrom))
{
nBytesReaded = fs.Read(aFirstBytes, 0, nBytesToRead);
}
return GetEncodingByContent(aFirstBytes, nBytesReaded);
}
private void GetEncodingAndDelimeter(string sFileFrom, out int? encoding, out int? delimiter)
{
encoding = null;
delimiter = null;
const int nBytesToRead = 1000;
byte[] aFirstBytes = new byte[nBytesToRead];
int nBytesReaded = 0;
using (FileStream fs = File.OpenRead(sFileFrom))
{
nBytesReaded = fs.Read(aFirstBytes, 0, nBytesToRead);
}
System.Text.Encoding oEncoding = GetEncodingByContent(aFirstBytes, nBytesReaded);
if (null != oEncoding)
encoding = oEncoding.CodePage;
else
encoding = System.Text.Encoding.UTF8.CodePage;
delimiter = GetDelimeterByContent(aFirstBytes, nBytesReaded);
}
private int? GetDelimeterByContent(byte[] aFirstBytes, int nBytesReaded)
{
int? delimiter = null;
int nLineLength = nBytesReaded;
int nDelimiter = (int)CsvDelimiter.None;
int nDelimitersCount = 6;
int[] aDelimiters = new int[nDelimitersCount];
for (int i = 0; i < nDelimitersCount; i++)
aDelimiters[i] = 0;
for (int i = 0; i < nBytesReaded; ++i)
{
byte cCurChar = aFirstBytes[i];
if ('\n' == cCurChar)
{
nLineLength = i;
break;
}
else if ('\t' == cCurChar)
aDelimiters[(int)CsvDelimiter.Tab]++;
else if (';' == cCurChar)
aDelimiters[(int)CsvDelimiter.Semicolon]++;
else if (':' == cCurChar)
aDelimiters[(int)CsvDelimiter.Сolon]++;
else if (',' == cCurChar)
aDelimiters[(int)CsvDelimiter.Comma]++;
else if (' ' == cCurChar)
aDelimiters[(int)CsvDelimiter.Space]++;
}
int nMaxVal = 0;
int nMaxIndex = 0;
for (int i = 1; i < nDelimitersCount; i++)
{
if (nMaxVal < aDelimiters[i])
{
nMaxVal = aDelimiters[i];
nMaxIndex = i;
}
}
if (nMaxVal > 0)
nDelimiter = nMaxIndex;
if ((int)CsvDelimiter.None != nDelimiter)
delimiter = nMaxIndex;
return delimiter;
}
private static System.Text.Encoding GetEncodingByContent(byte[] aFirstBytes, int nBytesReaded)
{
System.Text.Encoding oEncoding = null;
if (nBytesReaded >= 2)
{
if (aFirstBytes[0] == 0xFF && aFirstBytes[1] == 0xFE)
oEncoding = System.Text.Encoding.Unicode;
else if (aFirstBytes[0] == 0xFE && aFirstBytes[1] == 0xFF)
oEncoding = System.Text.Encoding.BigEndianUnicode;
}
if (nBytesReaded >= 3)
{
if (aFirstBytes[0] == 0xEF && aFirstBytes[1] == 0xBB && aFirstBytes[2] == 0xBF)
oEncoding = System.Text.Encoding.UTF8;
}
return oEncoding;
}
private void GCThread()
{
const double dSleepTimeOut = 5;
int nSleepCount = 0;
while (!_stopEvt.WaitOne(0))
{
try
{
if (nSleepCount <=0 && 0 == m_nRunThreadCount)
{
ITaskResultInterface oTaskResult = TaskResult.NewTaskResult();
List<TaskResultData> aTasts = null;
int nMaxCount = int.Parse(ConfigurationManager.AppSettings["fileconverterservice.gc.removedtaskatonce"] ?? "10");
ErrorTypes oError = oTaskResult.GetExpired(nMaxCount, out aTasts);
if (ErrorTypes.NoError == oError && aTasts != null && aTasts.Count > 0)
{
Storage oStorage = new Storage();
DocsChanges oDocsChanges = new DocsChanges();
foreach (TaskResultData oTast in aTasts)
{
if (ErrorTypes.NoError == oTaskResult.Remove(oTast.sKey))
{
oStorage.RemovePath(oTast.sKey);
oDocsChanges.RemoveChanges(oTast.sKey);
}
}
}
else
{
nSleepCount = (int)(double.Parse(ConfigurationManager.AppSettings["fileconverterservice.gc.runperiod"] ?? "3600") / dSleepTimeOut);
}
}
else
{
if(nSleepCount > 0)
nSleepCount--;
Thread.Sleep(TimeSpan.FromSeconds(dSleepTimeOut));
}
}
catch (Exception e)
{
_log.Error("Exception catched while garbage collector working.", e);
}
}
}
private void PercentUpdateCallback(Object stateInfo)
{
try
{
List<KeyValuePair<string, int>> mapTemp = new List<KeyValuePair<string,int>>();
lock (m_oKeyToPercentLock)
{
foreach (KeyValuePair<string, int> entry in m_mapKeyToPercent)
mapTemp.Add(entry);
m_mapKeyToPercent.Clear();
}
ITaskResultInterface oTaskResult = TaskResult.NewTaskResult();
for (int i = 0, length = mapTemp.Count; i < length; ++i)
{
KeyValuePair<string, int> elem = mapTemp[i];
lock (m_oKeyToPercentValidateLock)
{
if (m_mapKeyToPercentValidate.ContainsKey(elem.Key))
{
TaskResultDataToUpdate oTaskResultData = new TaskResultDataToUpdate();
oTaskResultData.nStatusInfo = elem.Value;
oTaskResult.Update(elem.Key, oTaskResultData);
}
}
}
lock (m_oKeyToPercentLock)
{
if (m_mapKeyToPercent.Count > 0)
{
int nPeriod = int.Parse(ConfigurationManager.AppSettings["fileconverterservice.percent.runperiod"] ?? "500");
m_oPercentTimer.Change(nPeriod, System.Threading.Timeout.Infinite);
}
else
{
m_oPercentTimer.Dispose();
m_oPercentTimer = null;
}
}
}
catch
{
m_oPercentTimer = null;
}
}
private static string EscapeCommandLineArguments(string arg)
{
return "\"" + arg.Replace("\\", "\\\\").Replace("\"", "\\\"") + "\"";
}
private ErrorTypes PostProcess(IddleProcess oIddleProcess, ErrorTypes eError, bool bNeedParam)
{
try
{
CTaskQueue oTaskQueue = new CTaskQueue();
ITaskResultInterface oTaskResult = TaskResult.NewTaskResult();
Storage oStorage = new Storage();
TaskQueueData oTaskQueueData = oIddleProcess.m_oTaskQueueData;
if (null != oTaskQueueData)
{
if (oIddleProcess.m_oProcess != null)
{
oIddleProcess.m_oProcess.CancelOutputRead();
}
if (ErrorTypes.NoError == eError && null != oIddleProcess.m_oProcess)
{
int nExitCode = oIddleProcess.m_oProcess.ExitCode;
_log.DebugFormat("ExitCode (code={0};id={1})", nExitCode, oTaskQueueData.m_sKey);
bool bUploadFile = true;
if (0 != nExitCode)
{
bUploadFile = false;
if (-(int)ErrorTypes.ConvertMS_OFFCRYPTO == nExitCode)
eError = ErrorTypes.ConvertMS_OFFCRYPTO;
else if (-(int)ErrorTypes.ConvertCorrupted == nExitCode)
{
eError = ErrorTypes.ConvertCorrupted;
bUploadFile = true;
}
else
eError = ErrorTypes.Convert;
SaveErrorFile(oIddleProcess.m_sDirToDelete, oTaskQueueData.m_sKey);
}
if (bUploadFile)
{
if (null != oIddleProcess.m_sDirResult)
{
ErrorTypes eErrorUpload = UploadDirectoryToStorage(oIddleProcess.m_sDirResult, oTaskQueueData.m_sKey, "", oStorage);
if (ErrorTypes.NoError != eErrorUpload)
eError = eErrorUpload;
}
_log.DebugFormat("UploadDirectoryToStorage complete(id={0})", oTaskQueueData.m_sKey);
}
}
lock (m_oKeyToPercentValidateLock)
{
m_mapKeyToPercentValidate.Remove(oTaskQueueData.m_sKey);
}
if (false == string.IsNullOrEmpty(oIddleProcess.m_sDirToDelete) && Directory.Exists(oIddleProcess.m_sDirToDelete))
Directory.Delete(oIddleProcess.m_sDirToDelete, true);
TaskResultDataToUpdate oTaskResultDataUpdate = new TaskResultDataToUpdate();
if (null != oIddleProcess.m_oTaskQueueData.m_sFromKey && !string.IsNullOrEmpty(oIddleProcess.m_oTaskQueueData.m_sFromKey))
oTaskResultDataUpdate.sTitle = Path.GetFileName(oIddleProcess.m_sFileTo);
if (true == bNeedParam)
{
oTaskResultDataUpdate.eStatus = FileStatus.NeedParams;
oTaskResult.Update(oTaskQueueData.m_sKey, oTaskResultDataUpdate);
_log.DebugFormat("oTaskResult.Update complete(status={0};id={1})", oTaskResultDataUpdate.eStatus, oTaskQueueData.m_sKey);
}
else
{
if (ErrorTypes.NoError == eError)
oTaskResultDataUpdate.eStatus = FileStatus.Ok;
else
{
if (ErrorTypes.ConvertDownload == eError)
oTaskResultDataUpdate.eStatus = FileStatus.ErrToReload;
else
oTaskResultDataUpdate.eStatus = FileStatus.Err;
}
oTaskResultDataUpdate.nStatusInfo = (int)eError;
oTaskResult.Update(oTaskQueueData.m_sKey, oTaskResultDataUpdate);
_log.DebugFormat("oTaskResult.Update complete(status={0};id={1})", oTaskResultDataUpdate.eStatus, oTaskQueueData.m_sKey);
}
if (null != oTaskQueueData.m_sResultCallbackUrl && !string.IsNullOrEmpty(oTaskQueueData.m_sResultCallbackUrl))
{
InputCommand cmd = InputCommand.DeserializeFromJson(oTaskQueueData.m_sResultCallbackData);
cmd.userid = oIddleProcess.m_sChangesAuthor;
string dataJson = InputCommand.SerializeToJson(cmd);
byte[] data = System.Text.Encoding.UTF8.GetBytes(dataJson);
AsyncWebRequestOperation oAsyncWebRequestOperation = new AsyncWebRequestOperation();
TransportClass oTransportClass = new TransportClass();
oTransportClass.m_oTaskQueueData = oTaskQueueData;
oTransportClass.m_oAsyncWebRequestOperation = oAsyncWebRequestOperation;
oTransportClass.m_oAsyncWebRequestOperationResult = oAsyncWebRequestOperation.RequestBegin(oTaskQueueData.m_sResultCallbackUrl, "POST", "application/json", data, RequestCallback, oTransportClass);
_log.DebugFormat("Request to url='{0}' data='{1}' begin(id={2})", oTaskQueueData.m_sResultCallbackUrl, dataJson, oTaskQueueData.m_sKey);
}
oTaskQueue.RemoveTask(oTaskQueueData.m_oDataKey);
_log.DebugFormat("oTaskResult.RemoveTask complete(id={0})", oTaskQueueData.m_sKey);
_log.DebugFormat("End Task(id={0})", oTaskQueueData.m_sKey);
}
}
catch(Exception e)
{
_log.Error("Exception catched in PostProcess:", e);
}
return eError;
}
private void RequestCallback(IAsyncResult ar)
{
TransportClass oTransportClass = ar.AsyncState as TransportClass;
try
{
byte[] aOutput;
ErrorTypes eError = oTransportClass.m_oAsyncWebRequestOperation.RequestEnd(oTransportClass.m_oAsyncWebRequestOperationResult, out aOutput);
if (_log.IsDebugEnabled)
{
try
{
_log.DebugFormat("RequestCallback Response='{0}'(id={1})", Encoding.UTF8.GetString(aOutput), oTransportClass.m_oTaskQueueData.m_sKey);
}
catch
{
}
}
}
catch (Exception e)
{
_log.Error("Exception catched in RequestCallback:", e);
}
}
private void SaveErrorFile(string sPath, string sKey)
{
string sErrorDir = ConfigurationSettings.AppSettings["fileconverterservice.converter.errorfiles"];
if (false == string.IsNullOrEmpty(sErrorDir) && false == string.IsNullOrEmpty(sPath))
UploadDirectoryToStorage(sPath, sErrorDir, "", new Storage());
}
private ErrorTypes UploadDirectoryToStorage(string sSourceDir, string sSourceFileStoreDir, string sLocalPath, Storage oFileStore)
{
ErrorTypes eResult = ErrorTypes.NoError;
string[] aFiles = Directory.GetFiles(Path.Combine(sSourceDir, sLocalPath));
for (int i = 0, length = aFiles.Length; i < length; ++i)
{
string sFile = aFiles[i];
string sFileStorePath = Path.Combine(sSourceFileStoreDir, sLocalPath);
sFileStorePath = Path.Combine(sFileStorePath, Path.GetFileName(sFile));
using (FileStream fs = new FileStream(sFile, FileMode.Open))
{
int nReadWriteBytes;
oFileStore.WriteFile(sFileStorePath, fs, out nReadWriteBytes);
}
}
string[] aDirectories = Directory.GetDirectories(Path.Combine(sSourceDir, sLocalPath));
for (int i = 0, length = aDirectories.Length; i < length; ++i)
{
string sDir = aDirectories[i];
string sDirLocal = Path.Combine(sLocalPath, Path.GetFileName(sDir));
string sDirStorePath = Path.Combine(sSourceFileStoreDir, sDirLocal);
oFileStore.CreateDirectory(sDirStorePath);
ErrorTypes eTypes = UploadDirectoryToStorage(sSourceDir, sSourceFileStoreDir, sDirLocal, oFileStore);
if (ErrorTypes.NoError != eTypes)
{
eResult = eTypes;
break;
}
}
return eResult;
}
ErrorTypes DownloadStorageTreeNodeToFilesystem(string sTargetDir, string sKey, string sLocalDir, StorageTreeNode oNode, Storage oFileStore)
{
ErrorTypes eError = ErrorTypes.NoError;
for (int i = 0, length = oNode.m_aSubNodes.Count; i < length; ++i)
{
StorageTreeNode oSubNode = oNode.m_aSubNodes[i];
string sLocalPath = Path.Combine(sLocalDir, oSubNode.m_sName);
if (oSubNode.m_bIsDirectory)
{
Directory.CreateDirectory(Path.Combine(sTargetDir, sLocalPath));
eError = DownloadStorageTreeNodeToFilesystem(sTargetDir, sKey, sLocalPath, oSubNode, oFileStore);
}
else
{
try
{
using (FileStream fs = new FileStream(Path.Combine(sTargetDir, sLocalPath), FileMode.Create))
{
int nReadWriteBytes;
oFileStore.ReadFile(Path.Combine(sKey, sLocalPath), fs, out nReadWriteBytes);
}
}
catch(Exception e)
{
eError = ErrorTypes.ConvertReadFile;
_log.Error("Exception catched in DownloadStorageTreeNodeToFilesystem:", e);
}
}
if (ErrorTypes.NoError != eError)
break;
}
return eError;
}
void convertProc_OutputDataReceived(object sender, System.Diagnostics.DataReceivedEventArgs e)
{
if (!String.IsNullOrEmpty(e.Data))
{
try
{
string[] aArgs = StringParser.ParseArguments(e.Data);
if (2 <= aArgs.Length)
{
int nPercent = System.Convert.ToInt32(aArgs[1].Replace("\"", ""));
string sKey = aArgs[0].Replace("\"", "");
lock (m_oKeyToPercentLock)
{
m_mapKeyToPercent[sKey] = nPercent;
if (null == m_oPercentTimer)
m_oPercentTimer = new Timer(PercentUpdateCallback, null, 0, System.Threading.Timeout.Infinite);
}
}
}
catch
{
}
}
}
}
}