inga-lovinde
/
FLocal
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

ShallerDBProcessor implemented

Browse Source
main
Inga 🏳‍🌈 15 years ago
parent 8b83a99e0d
commit 01aff7207f
10 changed files with 176 additions and 35 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      Builder/IISMainHandler/build.txt
  2. 2
      Builder/IISUploadHandler/build.txt
  3. 3
      Common/actions/InsertChange.cs
  4. 7
      Common/dataobjects/Board.cs
  5. 5
      Common/dataobjects/Post.cs
  6. 59
      Common/dataobjects/Thread.cs
  7. 1
      ImportConsole/ImportConsole.csproj
  8. 6
      ImportConsole/Program.cs
  9. 122
      ImportConsole/ShallerDBProcessor.cs
  10. 4
      ImportConsole/ThreadedHTMLProcessor.cs

2
Builder/IISMainHandler/build.txt
Unescape View File

@ -1 +1 @@
520
533

2
Builder/IISUploadHandler/build.txt
Unescape View File

@ -1 +1 @@
255
268

3
Common/actions/InsertChange.cs
Unescape View File

@ -35,6 +35,9 @@ namespace FLocal.Common.actions {
this.tableSpec,
processedData
));
if(processedData.ContainsKey(this.tableSpec.idName)) {
this.id = int.Parse(processedData[this.tableSpec.idName]);
}
}
}

7
Common/dataobjects/Board.cs
Unescape View File

@ -333,6 +333,9 @@ namespace FLocal.Common.dataobjects {
}
public Thread CreateThread(User poster, string title, string body, PostLayer desiredLayer) {
return this.CreateThread(poster, title, body, desiredLayer, DateTime.Now, null);
}
public Thread CreateThread(User poster, string title, string body, PostLayer desiredLayer, DateTime date, int? forcedPostId) {
PostLayer actualLayer = poster.getActualLayer(this, desiredLayer);
AbstractChange threadInsert = new InsertChange(
@ -347,7 +350,7 @@ namespace FLocal.Common.dataobjects {
{ Thread.TableSpec.FIELD_TOTALVIEWS, new ScalarFieldValue("0") },
{ Thread.TableSpec.FIELD_FIRSTPOSTID, new ScalarFieldValue(null) },
{ Thread.TableSpec.FIELD_LASTPOSTID, new ScalarFieldValue(null) },
{ Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) },
{ Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(date.ToUTCString()) },
}
);
@ -355,7 +358,7 @@ namespace FLocal.Common.dataobjects {
Config.Transactional(transaction => {
threadInsertSet.Add(threadInsert);
threadInsertSet.Apply(transaction);
foreach(AbstractChange change in Thread.getNewPostChanges(this, threadInsert.getId().Value, null, poster, actualLayer, title, body).Value) {
foreach(AbstractChange change in Thread.getNewPostChanges(this, threadInsert.getId().Value, null, poster, actualLayer, title, body, date, forcedPostId).Value) {
dataInsertSet.Add(change);
}
dataInsertSet.Apply(transaction);

5
Common/dataobjects/Post.cs
Unescape View File

@ -197,6 +197,9 @@ namespace FLocal.Common.dataobjects {
}
public Post Reply(User poster, string title, string body, PostLayer desiredLayer) {
return this.Reply(poster, title, body, desiredLayer, DateTime.Now, null);
}
public Post Reply(User poster, string title, string body, PostLayer desiredLayer, DateTime date, int? forcedPostId) {
if(this.thread.isLocked) {
throw new FLocalException("thread locked");
@ -204,7 +207,7 @@ namespace FLocal.Common.dataobjects {
PostLayer actualLayer = poster.getActualLayer(this.thread.board, desiredLayer);
var changes = Thread.getNewPostChanges(this.thread.board, this.threadId, this, poster, actualLayer, title, body);
var changes = Thread.getNewPostChanges(this.thread.board, this.threadId, this, poster, actualLayer, title, body, date, forcedPostId);
ChangeSetUtil.ApplyChanges(changes.Value.ToArray());
return Post.LoadById(changes.Key.getId().Value);

59
Common/dataobjects/Thread.cs
Unescape View File

@ -376,44 +376,57 @@ namespace FLocal.Common.dataobjects {
);
}
internal static KeyValuePair<AbstractChange, IEnumerable<AbstractChange>> getNewPostChanges(Board board, int threadId, Post parentPost, User poster, PostLayer layer, string title, string body) {
internal static KeyValuePair<AbstractChange, IEnumerable<AbstractChange>> getNewPostChanges(Board board, int threadId, Post parentPost, User poster, PostLayer layer, string title, string body, DateTime date, int? forcedPostId) {
string parentPostId = null;
if(parentPost != null) parentPostId = parentPost.id.ToString();
bool isNewThread = (parentPost == null);
string bodyIntermediate;
if(forcedPostId.HasValue) {
//dirty hack
bodyIntermediate = body;
} else {
bodyIntermediate = UBBParser.UBBToIntermediate(body);
}
var postInsertData = new Dictionary<string,AbstractFieldValue> {
{ Post.TableSpec.FIELD_THREADID, new ScalarFieldValue(threadId.ToString()) },
{ Post.TableSpec.FIELD_PARENTPOSTID, new ScalarFieldValue(parentPostId) },
{ Post.TableSpec.FIELD_POSTERID, new ScalarFieldValue(poster.id.ToString()) },
{ Post.TableSpec.FIELD_POSTDATE, new ScalarFieldValue(date.ToUTCString()) },
{ Post.TableSpec.FIELD_REVISION, new ScalarFieldValue("0") },
{ Post.TableSpec.FIELD_LASTCHANGEDATE, new ScalarFieldValue(date.ToUTCString()) },
{ Post.TableSpec.FIELD_LAYERID, new ScalarFieldValue(layer.id.ToString()) },
{ Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
{ Post.TableSpec.FIELD_BODY, new ScalarFieldValue(bodyIntermediate) },
};
if(forcedPostId.HasValue) {
postInsertData[Post.TableSpec.FIELD_ID] = new ScalarFieldValue(forcedPostId.Value.ToString());
}
AbstractChange postInsert = new InsertChange(
Post.TableSpec.instance,
new Dictionary<string,AbstractFieldValue> {
{ Post.TableSpec.FIELD_THREADID, new ScalarFieldValue(threadId.ToString()) },
{ Post.TableSpec.FIELD_PARENTPOSTID, new ScalarFieldValue(parentPostId) },
{ Post.TableSpec.FIELD_POSTERID, new ScalarFieldValue(poster.id.ToString()) },
{ Post.TableSpec.FIELD_POSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) },
{ Post.TableSpec.FIELD_REVISION, new ScalarFieldValue("0") },
{ Post.TableSpec.FIELD_LASTCHANGEDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) },
{ Post.TableSpec.FIELD_LAYERID, new ScalarFieldValue(layer.id.ToString()) },
{ Post.TableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
{ Post.TableSpec.FIELD_BODY, new ScalarFieldValue(UBBParser.UBBToIntermediate(body)) },
}
postInsertData
);
AbstractFieldValue postReference = new ReferenceFieldValue(postInsert);
AbstractFieldValue postIndexReference = new TwoWayReferenceFieldValue(postInsert, TwoWayReferenceFieldValue.GREATEST);
AbstractChange revisionInsert = new InsertChange(
Post.RevisionTableSpec.instance,
new Dictionary<string,AbstractFieldValue> {
{ Post.RevisionTableSpec.FIELD_POSTID, new ReferenceFieldValue(postInsert) },
{ Post.RevisionTableSpec.FIELD_POSTID, postReference },
{ Post.RevisionTableSpec.FIELD_NUMBER, new ScalarFieldValue("0") },
{ Post.RevisionTableSpec.FIELD_CHANGEDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) },
{ Post.RevisionTableSpec.FIELD_CHANGEDATE, new ScalarFieldValue(date.ToUTCString()) },
{ Post.RevisionTableSpec.FIELD_TITLE, new ScalarFieldValue(title) },
{ Post.RevisionTableSpec.FIELD_BODY, new ScalarFieldValue(body) },
}
);
Dictionary<string, AbstractFieldValue> threadData = new Dictionary<string,AbstractFieldValue> {
{ Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(DateTime.Now.ToUTCString()) },
{ Thread.TableSpec.FIELD_LASTPOSTDATE, new ScalarFieldValue(date.ToUTCString()) },
{ Thread.TableSpec.FIELD_TOTALPOSTS, new IncrementFieldValue() },
{
Thread.TableSpec.FIELD_LASTPOSTID,
new TwoWayReferenceFieldValue(postInsert, TwoWayReferenceFieldValue.GREATEST)
postIndexReference
}
};
if(isNewThread) {
threadData[Thread.TableSpec.FIELD_FIRSTPOSTID] = new ReferenceFieldValue(postInsert);
threadData[Thread.TableSpec.FIELD_FIRSTPOSTID] = postReference;
}
AbstractChange threadUpdate = new UpdateChange(
Thread.TableSpec.instance,
@ -439,17 +452,7 @@ namespace FLocal.Common.dataobjects {
{ Board.TableSpec.FIELD_TOTALPOSTS, new IncrementFieldValue() },
{
Board.TableSpec.FIELD_LASTPOSTID,
new TwoWayReferenceFieldValue(
postInsert,
(oldStringId, newStringId) => {
if((oldStringId == null) || (oldStringId == "")) {
return newStringId;
}
int oldId = int.Parse(oldStringId);
int newId = int.Parse(newStringId);
return Math.Max(oldId, newId).ToString();
}
)
postIndexReference
}
};
if(isNewThread) {

1
ImportConsole/ImportConsole.csproj
Unescape View File

@ -52,6 +52,7 @@
<ItemGroup>
<Compile Include="Program.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="ShallerDBProcessor.cs" />
<Compile Include="ThreadedHTMLProcessor.cs" />
<Compile Include="UploadProcessor.cs" />
<Compile Include="UsersImporter.cs" />

6
ImportConsole/Program.cs
Unescape View File

@ -43,5 +43,11 @@ namespace FLocal.ImportConsole {
public static void ConvertThreaded(string pathToThreaded, string outFile) {
ThreadedHTMLProcessor.Process(pathToThreaded, outFile);
}
[Action]
public static void ImportShallerDB(string pathToDB) {
initializeConfig();
ShallerDBProcessor.processDB(pathToDB);
}
}
}

122
ImportConsole/ShallerDBProcessor.cs
Unescape View File

@ -0,0 +1,122 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using FLocal.Core;
using FLocal.Importer;
using FLocal.Common.actions;
using FLocal.Common.dataobjects;
namespace FLocal.ImportConsole {
static class ShallerDBProcessor {
private readonly static Dictionary<int, string> discussions = new Dictionary<int, string> {
{ 384486, "Common.Photos" },
};
private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0);
public static void processDB(string filename) {
using(StreamReader reader = new StreamReader(filename)) {
int i=0;
while(!reader.EndOfStream) {
string line = reader.ReadLine().Trim();
if(line == "") {
continue;
}
if(i%1000 == 0) {
Console.Write("[" + (int)(i/1000) + "]");
}
Dictionary<string, string> data = DictionaryConverter.FromDump(line);
int postId = int.Parse(data["Number"]);
try {
bool exists = false;
try {
Post post = Post.LoadById(postId);
exists = true;
} catch(NotFoundInDBException) {
}
if(exists) {
Console.Write("-");
} else {
int localMain = int.Parse(data["Local_Main"]);
int main = int.Parse(data["Main"]);
DateTime date = UNIX.AddSeconds(int.Parse(data["UnixTime"])).ToLocalTime();
User user;
string username = data["Username"];
try {
user = User.LoadByName(username);
} catch(NotFoundInDBException) {
Console.WriteLine("Cannot find user " + username);
ChangeSetUtil.ApplyChanges(
new InsertChange(
User.TableSpec.instance,
new Dictionary<string, AbstractFieldValue> {
{ User.TableSpec.FIELD_NAME, new ScalarFieldValue(username) },
{ User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(date.ToUTCString()) },
{ User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue(User.ENUM_SHOWPOSTSTOUSERS_ALL) },
{ User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_LOCATION, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TITLE, new ScalarFieldValue("") },
{ User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") },
{ User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") },
}
)
);
user = User.LoadByName(data["Username"]);
}
string title = data["Subject"];
string body = data["Body"];
PostLayer layer = PostLayer.LoadById(1);
if(data.ContainsKey("Layer")) {
layer = PostLayer.LoadById(int.Parse(data["Layer"]));
}
if(postId == main || postId == localMain) {
//first post in the thread
string legacyBoardName;
if(localMain != 0) {
legacyBoardName = discussions[main];
} else {
legacyBoardName = data["Board"];
}
Board board;
try {
board = Board.LoadByLegacyName(legacyBoardName);
} catch(NotFoundInDBException) {
throw new ApplicationException("Cannot find board '" + legacyBoardName + "'");
}
board.CreateThread(user, title, body, layer, date, postId);
} else {
int parentId = int.Parse(data["Parent"]);
if(parentId == 0) {
parentId = localMain;
if(parentId == 0) {
parentId = main;
}
}
Post post;
try {
post = Post.LoadById(parentId);
} catch(NotFoundInDBException) {
throw new ApplicationException("Cannot find parent post #" + parentId);
}
post.Reply(user, title, body, layer, date, postId);
}
Console.Write("+");
}
} catch(Exception e) {
Console.Error.WriteLine("Cannot process post #" + postId + ": " + e.GetType().FullName + ": " + e.Message);
Console.Error.WriteLine(e.StackTrace);
Console.Write("!");
// Console.ReadLine();
} finally {
i++;
}
}
}
}
}
}

4
ImportConsole/ThreadedHTMLProcessor.cs
Unescape View File

@ -8,7 +8,7 @@ using System.Text.RegularExpressions;
namespace FLocal.ImportConsole {
class ThreadedHTMLProcessor {
private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0).ToLocalTime();
private readonly static DateTime UNIX = new DateTime(1970, 1, 1, 0, 0, 0);
private readonly static Regex PARENT_BEGINMARKER = new Regex("<font class=\"small\">\\[<a href=\"/a?showthreaded.php\\?Cat=&amp;Board=\\w+&amp;Number=");
private const string PARENT_ENDMARKER = "&amp;";
@ -304,7 +304,7 @@ namespace FLocal.ImportConsole {
new Dictionary<string, string> {
{ "Subject", contentTitle },
{ "Board", contentBoard },
{ "UnixTime", ((int)(contentDate.Subtract(UNIX).TotalSeconds)).ToString() },
{ "UnixTime", ((int)(contentDate.ToUniversalTime().Subtract(UNIX).TotalSeconds)).ToString() },
{ "Parent", contentParent.ToString() },
{ "Main", contentThread.ToString() },
{ "Local_Main", contentThread.ToString() },

Write Preview
Loading…
Cancel
Save