diff --git a/Builder/IISMainHandler/build.txt b/Builder/IISMainHandler/build.txt index e157b5b..c97fea1 100644 --- a/Builder/IISMainHandler/build.txt +++ b/Builder/IISMainHandler/build.txt @@ -1 +1 @@ -520 \ No newline at end of file +533 \ No newline at end of file diff --git a/Builder/IISUploadHandler/build.txt b/Builder/IISUploadHandler/build.txt index b8e817d..02225a5 100644 --- a/Builder/IISUploadHandler/build.txt +++ b/Builder/IISUploadHandler/build.txt @@ -1 +1 @@ -255 \ No newline at end of file +268 \ No newline at end of file diff --git a/Common/actions/InsertChange.cs b/Common/actions/InsertChange.cs index 304ac3d..cd3965e 100644 --- a/Common/actions/InsertChange.cs +++ b/Common/actions/InsertChange.cs @@ -35,6 +35,9 @@ namespace FLocal.Common.actions { this.tableSpec, processedData )); + if(processedData.ContainsKey(this.tableSpec.idName)) { + this.id = int.Parse(processedData[this.tableSpec.idName]); + } } } diff --git a/Common/dataobjects/Board.cs b/Common/dataobjects/Board.cs index ac68561..33bb749 100644 --- a/Common/dataobjects/Board.cs +++ b/Common/dataobjects/Board.cs @@ -333,6 +333,9 @@ namespace FLocal.Common.dataobjects { } public Thread CreateThread(User poster, string title, string body, PostLayer desiredLayer) { + return this.CreateThread(poster, title, body, desiredLayer, DateTime.Now, null); + } + public Thread CreateThread(User poster, string title, string body, PostLayer desiredLayer, DateTime date, int? forcedPostId) { PostLayer actualLayer = poster.getActualLayer(this, desiredLayer); AbstractChange threadInsert = new InsertChange( @@ -347,7 +350,7 @@ namespace FLocal.Common.dataobjects { { Thread.TableSpec.FIELD_TOTALVIEWS, new ScalarFieldValue("0") }, { Thread.TableSpec.FIELD_FIRSTPOSTID, new ScalarFieldValue(null) }, { Thread.TableSpec.FIELD_LASTPOSTID, new ScalarFieldValue(null) }, - { Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) }, + { Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(date.ToUTCString()) }, } ); @@ -355,7 +358,7 @@ namespace FLocal.Common.dataobjects { Config.Transactional(transaction => { threadInsertSet.Add(threadInsert); threadInsertSet.Apply(transaction); - foreach(AbstractChange change in Thread.getNewPostChanges(this, threadInsert.getId().Value, null, poster, actualLayer, title, body).Value) { + foreach(AbstractChange change in Thread.getNewPostChanges(this, threadInsert.getId().Value, null, poster, actualLayer, title, body, date, forcedPostId).Value) { dataInsertSet.Add(change); } dataInsertSet.Apply(transaction); diff --git a/Common/dataobjects/Post.cs b/Common/dataobjects/Post.cs index 8a36e7b..ab3f56d 100644 --- a/Common/dataobjects/Post.cs +++ b/Common/dataobjects/Post.cs @@ -197,6 +197,9 @@ namespace FLocal.Common.dataobjects { } public Post Reply(User poster, string title, string body, PostLayer desiredLayer) { + return this.Reply(poster, title, body, desiredLayer, DateTime.Now, null); + } + public Post Reply(User poster, string title, string body, PostLayer desiredLayer, DateTime date, int? forcedPostId) { if(this.thread.isLocked) { throw new FLocalException("thread locked"); @@ -204,7 +207,7 @@ namespace FLocal.Common.dataobjects { PostLayer actualLayer = poster.getActualLayer(this.thread.board, desiredLayer); - var changes = Thread.getNewPostChanges(this.thread.board, this.threadId, this, poster, actualLayer, title, body); + var changes = Thread.getNewPostChanges(this.thread.board, this.threadId, this, poster, actualLayer, title, body, date, forcedPostId); ChangeSetUtil.ApplyChanges(changes.Value.ToArray()); return Post.LoadById(changes.Key.getId().Value); diff --git a/Common/dataobjects/Thread.cs b/Common/dataobjects/Thread.cs index d629459..b10c4fd 100644 --- a/Common/dataobjects/Thread.cs +++ b/Common/dataobjects/Thread.cs @@ -376,44 +376,57 @@ namespace FLocal.Common.dataobjects { ); } - internal static KeyValuePair> getNewPostChanges(Board board, int threadId, Post parentPost, User poster, PostLayer layer, string title, string body) { + internal static KeyValuePair> getNewPostChanges(Board board, int threadId, Post parentPost, User poster, PostLayer layer, string title, string body, DateTime date, int? forcedPostId) { string parentPostId = null; if(parentPost != null) parentPostId = parentPost.id.ToString(); bool isNewThread = (parentPost == null); + string bodyIntermediate; + if(forcedPostId.HasValue) { + //dirty hack + bodyIntermediate = body; + } else { + bodyIntermediate = UBBParser.UBBToIntermediate(body); + } + var postInsertData = new Dictionary { + { Post.TableSpec.FIELD_THREADID, new ScalarFieldValue(threadId.ToString()) }, + { Post.TableSpec.FIELD_PARENTPOSTID, new ScalarFieldValue(parentPostId) }, + { Post.TableSpec.FIELD_POSTERID, new ScalarFieldValue(poster.id.ToString()) }, + { Post.TableSpec.FIELD_POSTDATE, new ScalarFieldValue(date.ToUTCString()) }, + { Post.TableSpec.FIELD_REVISION, new ScalarFieldValue("0") }, + { Post.TableSpec.FIELD_LASTCHANGEDATE, new ScalarFieldValue(date.ToUTCString()) }, + { Post.TableSpec.FIELD_LAYERID, new ScalarFieldValue(layer.id.ToString()) }, + { Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) }, + { Post.TableSpec.FIELD_BODY, new ScalarFieldValue(bodyIntermediate) }, + }; + if(forcedPostId.HasValue) { + postInsertData[Post.TableSpec.FIELD_ID] = new ScalarFieldValue(forcedPostId.Value.ToString()); + } AbstractChange postInsert = new InsertChange( Post.TableSpec.instance, - new Dictionary { - { Post.TableSpec.FIELD_THREADID, new ScalarFieldValue(threadId.ToString()) }, - { Post.TableSpec.FIELD_PARENTPOSTID, new ScalarFieldValue(parentPostId) }, - { Post.TableSpec.FIELD_POSTERID, new ScalarFieldValue(poster.id.ToString()) }, - { Post.TableSpec.FIELD_POSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) }, - { Post.TableSpec.FIELD_REVISION, new ScalarFieldValue("0") }, - { Post.TableSpec.FIELD_LASTCHANGEDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) }, - { Post.TableSpec.FIELD_LAYERID, new ScalarFieldValue(layer.id.ToString()) }, - { Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) }, - { Post.TableSpec.FIELD_BODY, new ScalarFieldValue(UBBParser.UBBToIntermediate(body)) }, - } + postInsertData ); + AbstractFieldValue postReference = new ReferenceFieldValue(postInsert); + AbstractFieldValue postIndexReference = new TwoWayReferenceFieldValue(postInsert, TwoWayReferenceFieldValue.GREATEST); AbstractChange revisionInsert = new InsertChange( Post.RevisionTableSpec.instance, new Dictionary { - { Post.RevisionTableSpec.FIELD_POSTID, new ReferenceFieldValue(postInsert) }, + { Post.RevisionTableSpec.FIELD_POSTID, postReference }, { Post.RevisionTableSpec.FIELD_NUMBER, new ScalarFieldValue("0") }, - { Post.RevisionTableSpec.FIELD_CHANGEDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) }, + { Post.RevisionTableSpec.FIELD_CHANGEDATE, new ScalarFieldValue(date.ToUTCString()) }, { Post.RevisionTableSpec.FIELD_TITLE, new ScalarFieldValue(title) }, { Post.RevisionTableSpec.FIELD_BODY, new ScalarFieldValue(body) }, } ); Dictionary threadData = new Dictionary { - { Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) }, + { Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(date.ToUTCString()) }, { Thread.TableSpec.FIELD_TOTALPOSTS, new IncrementFieldValue() }, { Thread.TableSpec.FIELD_LASTPOSTID, - new TwoWayReferenceFieldValue(postInsert, TwoWayReferenceFieldValue.GREATEST) + postIndexReference } }; if(isNewThread) { - threadData[Thread.TableSpec.FIELD_FIRSTPOSTID] = new ReferenceFieldValue(postInsert); + threadData[Thread.TableSpec.FIELD_FIRSTPOSTID] = postReference; } AbstractChange threadUpdate = new UpdateChange( Thread.TableSpec.instance, @@ -439,17 +452,7 @@ namespace FLocal.Common.dataobjects { { Board.TableSpec.FIELD_TOTALPOSTS, new IncrementFieldValue() }, { Board.TableSpec.FIELD_LASTPOSTID, - new TwoWayReferenceFieldValue( - postInsert, - (oldStringId, newStringId) => { - if((oldStringId == null) || (oldStringId == "")) { - return newStringId; - } - int oldId = int.Parse(oldStringId); - int newId = int.Parse(newStringId); - return Math.Max(oldId, newId).ToString(); - } - ) + postIndexReference } }; if(isNewThread) { diff --git a/ImportConsole/ImportConsole.csproj b/ImportConsole/ImportConsole.csproj index e912e44..9087bae 100644 --- a/ImportConsole/ImportConsole.csproj +++ b/ImportConsole/ImportConsole.csproj @@ -52,6 +52,7 @@ + diff --git a/ImportConsole/Program.cs b/ImportConsole/Program.cs index 537b94c..0208817 100644 --- a/ImportConsole/Program.cs +++ b/ImportConsole/Program.cs @@ -43,5 +43,11 @@ namespace FLocal.ImportConsole { public static void ConvertThreaded(string pathToThreaded, string outFile) { ThreadedHTMLProcessor.Process(pathToThreaded, outFile); } + + [Action] + public static void ImportShallerDB(string pathToDB) { + initializeConfig(); + ShallerDBProcessor.processDB(pathToDB); + } } } diff --git a/ImportConsole/ShallerDBProcessor.cs b/ImportConsole/ShallerDBProcessor.cs new file mode 100644 index 0000000..be24b8a --- /dev/null +++ b/ImportConsole/ShallerDBProcessor.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.IO; +using FLocal.Core; +using FLocal.Importer; +using FLocal.Common.actions; +using FLocal.Common.dataobjects; + +namespace FLocal.ImportConsole { + static class ShallerDBProcessor { + + private readonly static Dictionary discussions = new Dictionary { + { 384486, "Common.Photos" }, + }; + + private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0); + + public static void processDB(string filename) { + using(StreamReader reader = new StreamReader(filename)) { + int i=0; + while(!reader.EndOfStream) { + string line = reader.ReadLine().Trim(); + if(line == "") { + continue; + } + if(i%1000 == 0) { + Console.Write("[" + (int)(i/1000) + "]"); + } + Dictionary data = DictionaryConverter.FromDump(line); + int postId = int.Parse(data["Number"]); + try { + bool exists = false; + try { + Post post = Post.LoadById(postId); + exists = true; + } catch(NotFoundInDBException) { + } + if(exists) { + Console.Write("-"); + } else { + int localMain = int.Parse(data["Local_Main"]); + int main = int.Parse(data["Main"]); + DateTime date = UNIX.AddSeconds(int.Parse(data["UnixTime"])).ToLocalTime(); + User user; + string username = data["Username"]; + try { + user = User.LoadByName(username); + } catch(NotFoundInDBException) { + Console.WriteLine("Cannot find user " + username); + ChangeSetUtil.ApplyChanges( + new InsertChange( + User.TableSpec.instance, + new Dictionary { + { User.TableSpec.FIELD_NAME, new ScalarFieldValue(username) }, + { User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(date.ToUTCString()) }, + { User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue(User.ENUM_SHOWPOSTSTOUSERS_ALL) }, + { User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue("") }, + { User.TableSpec.FIELD_LOCATION, new ScalarFieldValue("") }, + { User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue("") }, + { User.TableSpec.FIELD_TITLE, new ScalarFieldValue("") }, + { User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") }, + { User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") }, + } + ) + ); + user = User.LoadByName(data["Username"]); + } + string title = data["Subject"]; + string body = data["Body"]; + PostLayer layer = PostLayer.LoadById(1); + if(data.ContainsKey("Layer")) { + layer = PostLayer.LoadById(int.Parse(data["Layer"])); + } + if(postId == main || postId == localMain) { + //first post in the thread + string legacyBoardName; + if(localMain != 0) { + legacyBoardName = discussions[main]; + } else { + legacyBoardName = data["Board"]; + } + Board board; + try { + board = Board.LoadByLegacyName(legacyBoardName); + } catch(NotFoundInDBException) { + throw new ApplicationException("Cannot find board '" + legacyBoardName + "'"); + } + board.CreateThread(user, title, body, layer, date, postId); + } else { + int parentId = int.Parse(data["Parent"]); + if(parentId == 0) { + parentId = localMain; + if(parentId == 0) { + parentId = main; + } + } + Post post; + try { + post = Post.LoadById(parentId); + } catch(NotFoundInDBException) { + throw new ApplicationException("Cannot find parent post #" + parentId); + } + post.Reply(user, title, body, layer, date, postId); + } + Console.Write("+"); + } + } catch(Exception e) { + Console.Error.WriteLine("Cannot process post #" + postId + ": " + e.GetType().FullName + ": " + e.Message); + Console.Error.WriteLine(e.StackTrace); + Console.Write("!"); +// Console.ReadLine(); + } finally { + i++; + } + } + } + } + + } +} diff --git a/ImportConsole/ThreadedHTMLProcessor.cs b/ImportConsole/ThreadedHTMLProcessor.cs index 652af45..fb1e421 100644 --- a/ImportConsole/ThreadedHTMLProcessor.cs +++ b/ImportConsole/ThreadedHTMLProcessor.cs @@ -8,7 +8,7 @@ using System.Text.RegularExpressions; namespace FLocal.ImportConsole { class ThreadedHTMLProcessor { - private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0).ToLocalTime(); + private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0); private readonly static Regex PARENT_BEGINMARKER = new Regex("\\[ { { "Subject", contentTitle }, { "Board", contentBoard }, - { "UnixTime", ((int)(contentDate.Subtract(UNIX).TotalSeconds)).ToString() }, + { "UnixTime", ((int)(contentDate.ToUniversalTime().Subtract(UNIX).TotalSeconds)).ToString() }, { "Parent", contentParent.ToString() }, { "Main", contentThread.ToString() }, { "Local_Main", contentThread.ToString() },