DisqusToStaticManComments/Program.cs at master · ChrisRisner/DisqusToStaticManComments · GitHub

History

224 lines (216 loc) · 10.9 KB

Raw

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

219

220

221

222

223

224

using System;

using System.IO;

using System.Xml;

using System.Linq;

using System.Collections.Generic;

using System.Xml.Linq;

using System.Text;

using System.Security.Cryptography;

namespace commentImporter

{

class Program

{

class ThreadObj {

public string ThreadId { get; set; }

public string ThreadName { get; set; }

public List<Post> Posts { get; set; }

}

class Post {

public string ParentPostId { get; set; }

public string ThreadTitle { get; set; }

public string ThreadId { get; set; }

public ThreadObj ParentThread { get; set; }

public string PostId { get; set; }

public string Parent { get; set; }

public List<Post> ChildrenPosts { get; set; }

public Post ParentPost { get; set; }

public string Message { get; set; }

public string Name { get; set; }

public string Email { get; set; }

public string Url { get; set; }

public string ReplyingTo { get; set; }

public string Hidden { get; set; }

public string Date { get; set; }

public string Tags { get; set; }

public string dateInMillis { get; set; }

public bool Processed { get; set; }

public string PostOrderDebug { get; set; }

public string GetFileText() {

StringBuilder sb = new StringBuilder();

sb.Append("_id: ");

sb.AppendLine(Guid.NewGuid().ToString());

sb.Append("_parent: /");

sb.AppendLine(this.ParentThread.ThreadName);

sb.Append("message: \"");

//Fix any characters that need escaping

sb.Append(this.Message.Replace("\\", "\\\\").Replace("\"", "\\\""));

if (this.Message.Contains("\"")) {

sb.Append("");

}

sb.AppendLine("\"");

sb.Append("name: ");

sb.AppendLine(this.Name.TrimStart('@'));

sb.Append("email: ");

sb.AppendLine(CalculateMD5Hash(this.Email));

sb.Append("url: '");

sb.Append(this.Url);

sb.AppendLine("'");

sb.Append("replying_to: '");

sb.Append(this.ReplyingTo);

sb.AppendLine("'");

sb.AppendLine("hidden: ''");

sb.Append("date: '");

sb.Append(this.Date);

sb.Append("'");

return sb.ToString();

}

/**

Used to calcualte MD5 hashes for hashing email addresses

public static string CalculateMD5Hash(string input)

{

// step 1, calculate MD5 hash from input

MD5 md5 = System.Security.Cryptography.MD5.Create();

byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input);

byte[] hash = md5.ComputeHash(inputBytes);

// step 2, convert byte array to hex string

StringBuilder sb = new StringBuilder();

for (int i = 0; i < hash.Length; i++)

{

sb.Append(hash[i].ToString("X2"));

}

return sb.ToString();

}

/***

Update any post children to be part of the same thread (only 1 deep replies)

static void processPostsChildren(Post post, int replyingTo) {

if (post.ChildrenPosts.Count > 0) {

foreach (var childPost in post.ChildrenPosts) {

childPost.ReplyingTo = replyingTo.ToString();

childPost.Processed = true;

processPostsChildren(childPost, replyingTo);

}

static void Main(string[] args)

{

Console.WriteLine("Starting Comment Processing");

//Create CommentFiles directory

if (!Directory.Exists("CommentFiles")) {

Directory.CreateDirectory("CommentFiles");

Console.WriteLine("CommentFiles directory created");

} else {

Console.WriteLine("CommentFiles directory already created");

}

//Read in comment XML from comments-source.xml file

XElement root = XElement.Load("comments-source.xml");

if (root != null) {

Console.WriteLine("Xml has been read");

} else {

Console.WriteLine("Xml was empty or could not be read");

}

XNamespace dsq = XNamespace.Get("http://disqus.com/disqus-internals");

Dictionary<string, string> threadNameAndIds = new Dictionary<string, string>();

List<Post> listPosts = new List<Post>();

List<ThreadObj> listThreads = new List<ThreadObj>();

foreach (var element in root.Elements()) {

if (element.Name.LocalName == "thread") {

//Process Thread elements

var threadId = element.Attribute(dsq + "id").Value;

Console.WriteLine("Processing thread: " + threadId);

var idElement = element.Elements().Where(e => e.Name.LocalName == "id").First();

var idElementValue = idElement.Value;

threadNameAndIds.Add(threadId, idElementValue);

listThreads.Add(new ThreadObj() { ThreadId = threadId, ThreadName = idElementValue, Posts = new List<Post>() });

//Create directories for threads

if (!Directory.Exists("CommentFiles/" + idElementValue)) {

Console.WriteLine("Creating directory: " + idElementValue);

Directory.CreateDirectory("CommentFiles/" + idElementValue);

}

} else if (element.Name.LocalName == "post") {

//Process Post elements

var post = new Post();

post.ChildrenPosts = new List<Post>();

post.Processed = false;

var postId = element.Attribute(dsq + "id").Value;

post.PostId = postId;

var postThreadElement = element.Elements().Where(e => e.Name.LocalName == "thread").First();

var postThreadId = postThreadElement.Attribute(dsq + "id").Value;

post.ThreadId = postThreadId;

Console.WriteLine("Processing Post: " + postId + " for thread: " + postThreadId + " with title: " + threadNameAndIds[postThreadId]);

var parentPostElement = element.Elements().Where(e => e.Name.LocalName == "parent").FirstOrDefault();

if (parentPostElement != null) {

var postParentPostId = parentPostElement.Attribute(dsq + "id").Value;

Console.WriteLine("With parent post: " + postParentPostId);

post.ParentPostId = postParentPostId;

post.ParentPost = listPosts.Where(p => p.PostId == postParentPostId).FirstOrDefault();

post.ParentPost.ChildrenPosts.Add(post);

}

var createdAtElement = element.Elements().Where(e => e.Name.LocalName == "createdAt").First();

post.Date = createdAtElement.Value;

var dt = DateTime.Parse(post.Date);

post.dateInMillis = new DateTimeOffset(dt).ToUnixTimeMilliseconds().ToString();

var authorElement = element.Elements().Where(e => e.Name.LocalName == "author").First();

var authorEmailElement = authorElement.Elements().Where(e => e.Name.LocalName == "email").First();

var authorNameElement = authorElement.Elements().Where(e => e.Name.LocalName == "name").First();

post.Email = authorEmailElement.Value;

post.Name = authorNameElement.Value;

var messageElement = element.Elements().Where(e => e.Name.LocalName == "message").First();

post.Message = messageElement.Value;

post.ParentThread = listThreads.Where(t => t.ThreadId == postThreadId).FirstOrDefault();

post.ParentThread.Posts.Add(post);

if (post.ParentThread.ThreadName == "Custom-Authentication-with-Azure-Mobile-Services-and-LensRocket") {

Console.WriteLine("DT: " + post.Date);

Console.WriteLine("MS: " + post.dateInMillis);

}

listPosts.Add(post);

}

//Update replyingTo values for all threaded Posts

foreach (var thread in listThreads) {

int replyingToCount = 1;

foreach (var post in thread.Posts.Where (p => p.ParentPost == null)) {

post.PostOrderDebug = replyingToCount.ToString();

if (post.ChildrenPosts.Count > 0) {

processPostsChildren(post, replyingToCount);

}

post.Processed = true;

replyingToCount++;

}

//Check to see if any posts were not processed

foreach (var post in listPosts) {

if (!post.Processed) {

Console.WriteLine("Post not processed");

}

//Create Post files

int threadCount = 0;

int postCount = 0;

foreach (var thread in listThreads) {

threadCount++;

foreach (var post in thread.Posts) {

postCount++;

if (!string.IsNullOrEmpty(thread.ThreadName)) {

File.AppendAllText("CommentFiles/" + thread.ThreadName + "/comment-" + post.dateInMillis + ".yml", post.GetFileText());

}

else {

Console.WriteLine("Empty ID: " + thread.ThreadId);

}

Console.WriteLine("Total threads: " + threadCount);

Console.WriteLine("Total posts: " + postCount);

var commentDirs = Directory.EnumerateDirectories("CommentFiles/");

//Clean up only empty directories

foreach (var commentDir in commentDirs) {

if (Directory.GetFiles(commentDir).Length == 0) {

Console.WriteLine("Removing directory: " + commentDir);

Directory.Delete(commentDir, true);

}

DisqusToStaticManComments/Program.cs at master · ChrisRisner/DisqusToStaticManComments · GitHub

Expand file tree

Breadcrumbs

Program.cs

Latest commit

History

Breadcrumbs

Program.cs

File metadata and controls