Users importer implemented;

15 years ago · 73fcaa7ae7
parent 1eee4ddff5
commit 73fcaa7ae7
17 changed files with 258 additions and 16 deletions
--- a/Builder/IISMainHandler/build.txt
+++ b/Builder/IISMainHandler/build.txt
@ -1 +1 @@
-231
+240
--- a/Builder/IISMainHandler/product.wxs
+++ b/Builder/IISMainHandler/product.wxs
@ -55,6 +55,12 @@
 			<Component Id="Importer.pdb" Guid="6AE31E67-EFC6-471B-BB55-0DF628399D35">
 				<File Id="Importer.pdb" Source="..\..\Importer\bin\Release\Importer.pdb" KeyPath="yes" Checksum="yes"/>
 			</Component>
+			<Component Id="ImportConsole.exe" Guid="45962809-9BE4-4792-9759-21C49FCA57F7">
+				<File Id="ImportConsole.exe" Source="..\..\ImportConsole\bin\Release\ImportConsole.exe" KeyPath="yes" Checksum="yes"/>
+			</Component>
+			<Component Id="ImportConsole.pdb" Guid="6E5DCF23-7247-4DE0-BB6D-1A8463723437">
+				<File Id="ImportConsole.pdb" Source="..\..\ImportConsole\bin\Release\ImportConsole.pdb" KeyPath="yes" Checksum="yes"/>
+			</Component>
 			<!--Component Id="MySql.Data.dll" Guid="90646729-6B21-4BEE-94C3-F39F5288EFB3">
 				<File Id="MySql.Data.dll" Source="..\..\ThirdParty\mysql-connector-net-6.2.3\MySql.Data\Provider\bin\Release\MySql.Data.dll" KeyPath="yes" Checksum="yes"/>
 			</Component>
@ -77,10 +83,16 @@
 			<ComponentRef Id='IISMainHandler.dll' />
 			<ComponentRef Id='IISMainHandler.pdb' />
 		</Feature>
-		<Feature Id='Importer' Title='Importer' Level='1'>
+		<Feature Id='Import' Title='Import utilities' Level='1'>
+			<Feature Id='Importer' Title='Importer library' Level='1'>
 				<ComponentRef Id='Importer.dll' />
 				<ComponentRef Id='Importer.pdb' />
 			</Feature>
+			<Feature Id='ImportConsole' Title='Import console' Level='1'>
+				<ComponentRef Id='ImportConsole.exe' />
+				<ComponentRef Id='ImportConsole.pdb' />
+			</Feature>
+		</Feature>
 		<Feature Id='Core' Title='Core library' Level='1'>
 			<ComponentRef Id='Core.dll' />
 			<ComponentRef Id='Core.pdb' />
--- a/Common/actions/AbstractChange.cs
+++ b/Common/actions/AbstractChange.cs
@ -5,7 +5,7 @@ using System.Text;
 using FLocal.Core.DB;

 namespace FLocal.Common.actions {
-	abstract class AbstractChange {
+	abstract public class AbstractChange {

 		private bool isApplied;

--- a/Common/actions/AbstractFieldValue.cs
+++ b/Common/actions/AbstractFieldValue.cs
@ -4,7 +4,7 @@ using System.Linq;
 using System.Text;

 namespace FLocal.Common.actions {
-	abstract class AbstractFieldValue {
+	abstract public class AbstractFieldValue {

 		abstract public string getStringRepresentation();
 		abstract public string getStringRepresentation(string oldInfo);
--- a/Common/actions/ChangeSet.cs
+++ b/Common/actions/ChangeSet.cs
@ -20,6 +20,7 @@ namespace FLocal.Common.actions {
 					tablesLockOrder_locker,
 					() => new List<string>() {
 						"Accounts",
+						"Users",
 						"Boards",
 						"Threads",
 						"Posts",
--- a/Common/actions/ChangeSetUtil.cs
+++ b/Common/actions/ChangeSetUtil.cs
@ -5,9 +5,9 @@ using System.Text;
 using FLocal.Core.DB;

 namespace FLocal.Common.actions {
-	class ChangeSetUtil {
+	public class ChangeSetUtil {

-		public static void WithChangeSet(Action<ChangeSet, Transaction> action) {
+		internal static void WithChangeSet(Action<ChangeSet, Transaction> action) {
 			using(ChangeSet changeSet = new ChangeSet()) {
 				Config.Transactional(transaction => {
 					action(changeSet, transaction);
@ -16,11 +16,11 @@ namespace FLocal.Common.actions {
 			}
 		}

-		public static void WithChangeSet(Action<ChangeSet> action) {
+		internal static void WithChangeSet(Action<ChangeSet> action) {
 			WithChangeSet((changeset, transaction) => action(changeset));
 		}

-		public static void ApplyChanges(IEnumerable<AbstractChange> changes) {
+		public static void ApplyChanges(params AbstractChange[] changes) {
 			using(ChangeSet changeSet = new ChangeSet()) {
 				foreach(AbstractChange change in changes) {
 					changeSet.Add(change);
--- a/Common/actions/InsertChange.cs
+++ b/Common/actions/InsertChange.cs
@ -5,7 +5,7 @@ using System.Text;
 using FLocal.Core.DB;

 namespace FLocal.Common.actions {
-	class InsertChange : AbstractChange {
+	public class InsertChange : AbstractChange {

 		private int? id;

--- a/Common/actions/ReferenceFieldValue.cs
+++ b/Common/actions/ReferenceFieldValue.cs
@ -4,7 +4,7 @@ using System.Linq;
 using System.Text;

 namespace FLocal.Common.actions {
-	class ReferenceFieldValue : AbstractFieldValue {
+	public class ReferenceFieldValue : AbstractFieldValue {

 		public readonly AbstractChange referenced;

--- a/Common/actions/ScalarFieldValue.cs
+++ b/Common/actions/ScalarFieldValue.cs
@ -4,7 +4,7 @@ using System.Linq;
 using System.Text;

 namespace FLocal.Common.actions {
-	class ScalarFieldValue : AbstractFieldValue {
+	public class ScalarFieldValue : AbstractFieldValue {

 		private readonly string scalar;

--- a/Common/dataobjects/User.cs
+++ b/Common/dataobjects/User.cs
@ -21,6 +21,7 @@ namespace FLocal.Common.dataobjects {
 			public const string FIELD_NAME = "Name";
 			public const string FIELD_USERGROUPID = "UserGroupId";
 			public const string FIELD_SHOWPOSTSTOUSERS = "ShowPostsToUsers";
+			public const string FIELD_BIOGRAPHY = "Biography";
 			public static readonly TableSpec instance = new TableSpec();
 			public string name { get { return TABLE; } }
 			public string idName { get { return FIELD_ID; } }
@ -93,6 +94,14 @@ namespace FLocal.Common.dataobjects {
 			}
 		}

+		private string _biography;
+		public string biography {
+			get {
+				this.LoadIfNotLoaded();
+				return this._biography;
+			}
+		}
+
 		private static Dictionary<string, int> id2user = new Dictionary<string,int>();
 		public static User LoadByName(string name) {
 			if(!id2user.ContainsKey(name)) {
@ -130,6 +139,7 @@ namespace FLocal.Common.dataobjects {
 			this._name = data[TableSpec.FIELD_NAME];
 			this._userGroupId = int.Parse(data[TableSpec.FIELD_USERGROUPID]);
 			this._showPostsToUsers = data[TableSpec.FIELD_SHOWPOSTSTOUSERS];
+			this._biography = data[TableSpec.FIELD_BIOGRAPHY];
 		}

 		public XElement exportToXmlForViewing(UserContext context) {
--- a/FLocal.sln
+++ b/FLocal.sln
@ -15,6 +15,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "tmp", "tmp\tmp.csproj", "{B
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Importer", "Importer\Importer.csproj", "{E2ECF86C-C0BA-4782-A485-E267BB213BB2}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ImportConsole", "ImportConsole\ImportConsole.csproj", "{208B84AF-A5DD-4C20-83D5-160EA243BA43}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@ -65,6 +67,12 @@ Global
 		{E2ECF86C-C0BA-4782-A485-E267BB213BB2}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{E2ECF86C-C0BA-4782-A485-E267BB213BB2}.Release|Any CPU.Build.0 = Release|Any CPU
 		{E2ECF86C-C0BA-4782-A485-E267BB213BB2}.Release|x86.ActiveCfg = Release|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Release|Any CPU.Build.0 = Release|Any CPU
+		{208B84AF-A5DD-4C20-83D5-160EA243BA43}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/IISMainHandler/handlers/request/MigrateAccountHandler.cs
+++ b/IISMainHandler/handlers/request/MigrateAccountHandler.cs
@ -30,7 +30,7 @@ namespace FLocal.IISHandler.handlers.request {
 		protected override XElement[] Do(WebContext context) {
 			Account account = Account.LoadById(int.Parse(context.httprequest.Form["accountId"]));
 			if(!account.needsMigration) throw new FLocalException("Already migrated");
-			string userInfo = ShallerGateway.getUserInfo(account.user.name);
+			string userInfo = ShallerGateway.getUserInfoAsString(account.user.name);
 			Regex regex = new Regex("\\(fhn\\:([a-z0-9]+)\\)", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Singleline);
 			Match match = regex.Match(userInfo);
 			if(!match.Success) throw new FLocalException("not found");
--- a/ImportConsole/ImportConsole.csproj
+++ b/ImportConsole/ImportConsole.csproj
@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProductVersion>9.0.30729</ProductVersion>
+    <SchemaVersion>2.0</SchemaVersion>
+    <ProjectGuid>{208B84AF-A5DD-4C20-83D5-160EA243BA43}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>FLocal.ImportConsole</RootNamespace>
+    <AssemblyName>ImportConsole</AssemblyName>
+    <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="System" />
+    <Reference Include="System.configuration" />
+    <Reference Include="System.Core">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Xml.Linq">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Data.DataSetExtensions">
+      <RequiredTargetFramework>3.5</RequiredTargetFramework>
+    </Reference>
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\Common\Common.csproj">
+      <Project>{CE888859-9E46-41F7-91CE-8EC106F3A625}</Project>
+      <Name>Common</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\Core\Core.csproj">
+      <Project>{6F532626-E9F8-498E-9683-1538E7CD62CB}</Project>
+      <Name>Core</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\Importer\Importer.csproj">
+      <Project>{E2ECF86C-C0BA-4782-A485-E267BB213BB2}</Project>
+      <Name>Importer</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
--- a/ImportConsole/Program.cs
+++ b/ImportConsole/Program.cs
@ -0,0 +1,56 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using FLocal.Importer;
+using FLocal.Common.actions;
+using FLocal.Common.dataobjects;
+using FLocal.Core;
+using System.Configuration;
+
+namespace FLocal.ImportConsole {
+	class Program {
+		static void Main(string[] args) {
+
+			if(!FLocal.Common.Config.isInitialized) {
+				lock(typeof(FLocal.Common.Config)) {
+					if(!FLocal.Common.Config.isInitialized) {
+						FLocal.Common.Config.Init(ConfigurationManager.AppSettings);
+					}
+				}
+			}
+
+			for(int i=1; i<800; i++) {
+				Console.Write("[" + i + "]");
+				foreach(string userName in ShallerGateway.getUserNames(i)) {
+					Console.Write(".");
+					Dictionary<string, string> userData = ShallerGateway.getUserInfo(userName);
+					AbstractChange addUser = new InsertChange(
+						User.TableSpec.instance,
+						new Dictionary<string, AbstractFieldValue>() {
+							{ User.TableSpec.FIELD_NAME, new ScalarFieldValue(userName) },
+							{ User.TableSpec.FIELD_REGDATE, new ScalarFieldValue(DateTime.Parse(userData["regDate"]).ToUTCString()) },
+							{ User.TableSpec.FIELD_LOCATION, new ScalarFieldValue(userData["location"]) },
+							{ User.TableSpec.FIELD_SHOWPOSTSTOUSERS, new ScalarFieldValue("All") },
+							{ User.TableSpec.FIELD_SIGNATURE, new ScalarFieldValue(userData["signature"]) },
+							{ User.TableSpec.FIELD_TITLE, new ScalarFieldValue(userData["title"]) },
+							{ User.TableSpec.FIELD_TOTALPOSTS, new ScalarFieldValue("0") },
+							{ User.TableSpec.FIELD_USERGROUPID, new ScalarFieldValue("1") },
+							{ User.TableSpec.FIELD_BIOGRAPHY, new ScalarFieldValue(userData["biography"]) },
+						}
+					);
+					AbstractChange addAccount = new InsertChange(
+						Account.TableSpec.instance,
+						new Dictionary<string,AbstractFieldValue>() {
+							{ Account.TableSpec.FIELD_NAME, new ScalarFieldValue(userName) },
+							{ Account.TableSpec.FIELD_NEEDSMIGRATION, new ScalarFieldValue("1") },
+							{ Account.TableSpec.FIELD_PASSWORDHASH, new ScalarFieldValue("*") },
+							{ Account.TableSpec.FIELD_USERID, new ReferenceFieldValue(addUser) },
+						}
+					);
+					ChangeSetUtil.ApplyChanges(addUser, addAccount);
+				}
+			}
+		}
+	}
+}
--- a/ImportConsole/Properties/AssemblyInfo.cs
+++ b/ImportConsole/Properties/AssemblyInfo.cs
@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("ImportConsole")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("Microsoft")]
+[assembly: AssemblyProduct("ImportConsole")]
+[assembly: AssemblyCopyright("Copyright © Microsoft 2010")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("6c8ecdee-da08-44af-b450-2af25abf48cf")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
--- a/Importer/ShallerGateway.cs
+++ b/Importer/ShallerGateway.cs
@ -2,13 +2,58 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
+using System.Web;

 namespace FLocal.Importer {
 	public class ShallerGateway {

-		public static string getUserInfo(string userName) {
+		public static string getUserInfoAsString(string userName) {
 			return ShallerConnector.getPageContent("showprofile.php?User=" + userName + "&What=login&showlite=l", new Dictionary<string,string>(), new System.Net.CookieContainer());
 		}

+		private static Dictionary<string, Regex> regexInfoCache = new Dictionary<string, Regex>();
+		private static Regex getInfoRegexByCaption(string caption) {
+			if(!regexInfoCache.ContainsKey(caption)) {
+				lock(caption) {
+					if(!regexInfoCache.ContainsKey(caption)) {
+						regexInfoCache[caption] = new Regex("<td[^>]*>\\s*" + caption + "\\s*</td>\\s*<td>\\s*([^<>]*)\\s*</td>", RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline);
+					}
+				}
+			}
+			return regexInfoCache[caption];
+		}
+
+		private static Dictionary<string, string> userImportStructure {
+			get {
+				return new Dictionary<string,string>() {
+					{ "regDate", "Дата\\s+регистрации" },
+					{ "signature", "Подпись" },
+					{ "title", "Титул" },
+					{ "location", "Расположение" },
+					{ "biography", "Биография" },
+				};
+			}
+		}
+
+		public static Dictionary<string, string> getUserInfo(string userName) {
+			string content = getUserInfoAsString(userName);
+			return userImportStructure.ToDictionary<KeyValuePair<string, string>, string, string>(
+				kvp => kvp.Key,
+				kvp => HttpUtility.HtmlDecode(getInfoRegexByCaption(kvp.Value).Match(content).Groups[1].Value).Trim()
+			);
+		}
+
+		public static IEnumerable<string> getUserNames(int pageNum) {
+			string content = ShallerConnector.getPageContent("showmembers.php?Cat=&sb=13&page=" + pageNum + "&showlite=l", new Dictionary<string,string>(), new System.Net.CookieContainer());
+			Regex matcher = new Regex(";User=([^&]+)&", RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.IgnoreCase);
+			MatchCollection matches = matcher.Matches(content);
+			HashSet<string> result = new HashSet<string>();
+			foreach(Match match in matches) {
+				result.Add(match.Groups[1].Value);
+			}
+			return result;
+		}
+
 	}
 }
--- a/MySQLConnector/PostgresDBTraits.cs
+++ b/MySQLConnector/PostgresDBTraits.cs
@ -22,7 +22,7 @@ namespace FLocal.MySQLConnector {
 					newCommand.Transaction = command.Transaction;
 				}
 				newCommand.CommandType = System.Data.CommandType.Text;
-				newCommand.CommandText = "SELECT CURRVAL(" + this.escapeIdentifier(sequenceName) + ")";
+				newCommand.CommandText = "SELECT CURRVAL('" + this.escapeIdentifier(sequenceName) + "')";
 				return (long)newCommand.ExecuteScalar();
 			}
 		}