Scraping answers and comments with a Stack Snippet

Question

I would like to make answer-dependent challenges work. To that end, I suggested including a Stack Snippet in the question which gives an overview over the existing questions as well as some other useful meta information, and maybe a leaderboard (details depend on the particular challenge).

The main thing I want to do for my own challenge is get some information from all answer bodies (you can assume a fixed answer format), and check that a certain comment is present (let's say "Correctness verified").

So what's the best way to scrape answer and comments in a Stack Snippet? I suppose I should request the page with an additional web request, because the Snippet can probably not access the DOM of the page. Other things to keep in mind are pagination and collapsed comment.

Also how could I test such a Snippet?

(I'm kinda hoping, this question here will produce some nice reusable answers for future challenges.)

Answer 1

Sort answers by score snippet

This is a version 1 2 of a snippet that can sort and list all answers to a question in increasing order of the score.

This snippet recognizes various different style of answer headings, but suggestions welcome for the ones which are missed out by the snippet.

The script will filter out answers who do not have a heading with a proper score.

Many thanks to @Martin Büttner for help with regex

[Update] New features in version 2:

• Added the accepted answer tick mark
• Added permalink for the answer
• Added the answered relative and UTC time information
• Fixed the shouldHaveScore filter's regex

Run the snippet below to see the answers of this question sorted by score.

var QUESTION_ID = 3532; // Obtain this from the url
// It will be like http://XYZ.stackexchange.com/questions/QUESTION_ID/... on any question page

var answers = [], page = 1;

var SCORE_REG = /\d+(?=[^\d&]*(?:<(?:s>[^&]*<\/s>|[^&]+>)[^\d&]*)*$)/;

function url(index) {
  return "http://api.stackexchange.com/2.2/questions/" +  QUESTION_ID + "/answers?page=" + index + "&pagesize=100&order=desc&sort=votes&site=codegolf&filter=!t)IWYnsLAZle2tQ3KqrVveCRJfxcRLe";
}

function getAnswers() {
  $.ajax({
    url: url(page++),
    method: "get",
    dataType: "jsonp",
    crossDomain: true,
    success: function (data) {
      answers.push.apply(answers, data.items);
      if (data.has_more) getAnswers();
      else process()
    }
  });
}

getAnswers();

function shouldHaveHeading(a) {
  var pass = false;
  try {
    pass |= /^(#|<h).*/.test(a.body_markdown);
    pass |= ["-", "="]
              .indexOf(a.body_markdown.split("\n")[1][0]) > -1;
  } catch (ex) {}
  return pass;
}

function shouldHaveScore(a) {
  var pass = false;
  try {
    pass |= SCORE_REG.test(a.body_markdown.split("\n")[0]);
  } catch (ex) {}
  return pass;
}

function getRelDate(previous) {

  var current = Date.now();
  var msPerMinute = 60 * 1000;
  var msPerHour = msPerMinute * 60;
  var msPerDay = msPerHour * 24;
  var msPerMonth = msPerDay * 30;
  var msPerYear = msPerDay * 365;

  var elapsed = current - previous;

  if (elapsed < msPerMinute) {
    return Math.round(elapsed/1000) + ' seconds ago';
  }

  if (elapsed < msPerHour) {
    return Math.round(elapsed/msPerMinute) + ' minutes ago';
  }

  if (elapsed < msPerDay ) {
    return Math.round(elapsed/msPerHour ) + ' hours ago';
  }

  if (elapsed < msPerMonth) {
    return 'approx. ' + Math.round(elapsed/msPerDay) + ' days ago';
  }

  if (elapsed < msPerYear) {
    return 'approx. ' + Math.round(elapsed/msPerMonth) + ' months ago';
  }

  return 'approx. ' + Math.round(elapsed/msPerYear ) + ' years ago';

}

function process() {
  answers = answers.filter(shouldHaveHeading);
  answers = answers.filter(shouldHaveScore);
  answers.sort(function (a, b) {
    var aB = +(a.body_markdown.split("\n")[0].match(SCORE_REG) || [Infinity])[0],
        bB = +(b.body_markdown.split("\n")[0].match(SCORE_REG) || [Infinity])[0];
    return aB - bB
  });
  answers.forEach(function (a) {
    var answer = $("#answer-template").html();
    answer = answer.replace("{{BODY}}", a.body)
                   .replace("{{NAME}}", a.owner.display_name)
                   .replace("{{REP}}", a.owner.reputation)
                   .replace("{{VOTES}}", a.score)
                   .replace("{{DATE}}", new Date(a.creation_date*1e3).toUTCString())
                   .replace("{{REL_TIME}}", getRelDate(a.creation_date*1e3))
                   .replace("{{POST_LINK}}", a.share_link)
                   .replace(/{{USER_LINK}}/g, a.owner.link)
                   .replace('{{img}}=""', "src=\"" + a.owner.profile_image + '"');
    answer = $(answer);
    if (a.is_accepted) {
      answer.find(".vote-accepted-on").removeAttr("style");
    }
    $("#answers").append(answer);
  });
}

body { text-align: left !important}

<script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.1/jquery.min.js"></script>
<link rel="stylesheet" type="text/css" href="//cdn.sstatic.net/codegolf/all.css?v=83c949450c8b">
<div id="answers"></div>
<div id="answer-template" style="display: none">
<div class="answer" >
  <table>
    <tbody><tr>
      <td class="votecell">
        <div class="vote">
          <span style="cursor: pointer;" title="Total Votes" itemprop="upvoteCount" class="vote-count-post ">{{VOTES}}</span>
          <span style="display: none" class="vote-accepted-on load-accepted-answer-date" title="The question owner accepted this as the best answer">accepted</span>
        </div>
      </td>
      <td class="answercell">
        <div class="post-text" itemprop="text">{{BODY}}</div>
        <table class="fw">
          <tbody><tr>
            <td class="vt">
              <div class="post-menu">
                <a href="{{POST_LINK}}" title="short permalink to this answer" class="short-link">share</a>
              </div>
            </td>
            <td class="post-signature" align="right">
              <div class="user-info ">
                <div class="user-action-time">answered <span title="{{DATE}}" class="relativetime">{{REL_TIME}}</span>
                </div>
                <div class="user-gravatar32">
                </div>
                <div class="user-details">
                  <br>
                </div>
              </div>
            </td>
            <td class="post-signature" align="right">   
              <div class="user-info user-hover">
                <div class="user-action-time">
                </div>
                <div class="user-gravatar32">
                  <a href="{{USER_LINK}}">
                    <div class="gravatar-wrapper-32">
                      <img {{img}} alt="" height="32" width="32">
                    </div>
                  </a>
                </div>
                <div class="user-details">
                  <a href="{{USER_LINK}}">{{NAME}}</a><br>
                  <span class="reputation-score" title="reputation score " dir="ltr">{{REP}}</span>
                </div>
              </div>
            </td>
            </tr>
          </tbody>
        </table>
      </td>
      </tr>
    </tbody>
  </table>
</div>
</div>

NOTE: Before trying to reuse the snippet, do make sure to change the value of the QUESTION_ID variable at the top of the JS code.

Answer 2

Use the Stack Exchange API to collect the data: http://api.stackexchange.com/
Then you don't have to access and parse the DOM, you'll only have to parse the answers.

Testing can be done at the Stack Snippets Sandbox on Meta Stack Overflow. You could create a snippet and test it there on one of the existing answer-dependent challenges.