[Yanel-commits] rev 23376 - in
public/yanel/trunk/src/resources/add-realm: src/build
src/java/org/wyona/yanel/impl/resources xslt
simon at wyona.com
simon at wyona.com
Thu Mar 22 23:27:53 CET 2007
Author: simon
Date: 2007-03-22 23:27:51 +0100 (Thu, 22 Mar 2007)
New Revision: 23376
Modified:
public/yanel/trunk/src/resources/add-realm/src/build/dependencies.xml
public/yanel/trunk/src/resources/add-realm/src/java/org/wyona/yanel/impl/resources/AddRealmResource.java
public/yanel/trunk/src/resources/add-realm/xslt/add-realm.xsl
Log:
adds scope fixes bug #5239, also adds maxSize to unlimited. thanks to josias!
Modified: public/yanel/trunk/src/resources/add-realm/src/build/dependencies.xml
===================================================================
--- public/yanel/trunk/src/resources/add-realm/src/build/dependencies.xml 2007-03-22 21:41:20 UTC (rev 23375)
+++ public/yanel/trunk/src/resources/add-realm/src/build/dependencies.xml 2007-03-22 22:27:51 UTC (rev 23376)
@@ -14,7 +14,7 @@
<artifact:dependencies pathId="maven2.resource.classpath" filesetId="maven2.resource.fileset">
<remoteRepository refid="wyona.remote.repository"/>
- <dependency groupId="wyona-org-crawler" artifactId="wyona-org-crawler" version="0.0.1-dev-r23281"/>
+ <dependency groupId="wyona-org-crawler" artifactId="wyona-org-crawler" version="0.0.1-dev-r23346"/>
<dependency groupId="apache-jakarta-commons" artifactId="apache-jakarta-commons-io" version="1.2"/>
</artifact:dependencies>
Modified: public/yanel/trunk/src/resources/add-realm/src/java/org/wyona/yanel/impl/resources/AddRealmResource.java
===================================================================
--- public/yanel/trunk/src/resources/add-realm/src/java/org/wyona/yanel/impl/resources/AddRealmResource.java 2007-03-22 21:41:20 UTC (rev 23375)
+++ public/yanel/trunk/src/resources/add-realm/src/java/org/wyona/yanel/impl/resources/AddRealmResource.java 2007-03-22 22:27:51 UTC (rev 23376)
@@ -21,6 +21,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.lenya.search.crawler.DumpingCrawler;
+import websphinx.DownloadParameters;
import org.apache.log4j.Category;
import java.io.BufferedReader;
@@ -134,7 +135,7 @@
transformer = TransformerFactory.newInstance().newTransformer(new StreamSource(XSLTFile));
// Add HashMap keys with dummy values for form fields
- String[] parameterNames = { "realmid", "realmname", "url", "fslocation", "crawldepth", "crawlmaxpages" };
+ String[] parameterNames = { "realmid", "realmname", "url", "fslocation", "crawldepth", "crawlmaxpages", "scope" };
for (int i=0; i<parameterNames.length; i++) {
String property = getConfiguration().getProperty(parameterNames[i]);
boolean propertyExists = getConfiguration().containsKey(parameterNames[i]);
@@ -159,7 +160,7 @@
parameterName = (String) keysIterator.next();
parameter = HttpServletRequestHelper.getParameter(request, parameterName);
- if (parameterName == "fslocation" || ("fslocation").equals(parameterName)) {
+ if (parameterName.equals("fslocation") || parameterName.equals("scope")) {
parameters.put(parameterName, parameter);
transformer.setParameter(parameterName, parameters.get(parameterName).toString());
} else if (parameter == null || ("").equals(parameter)) {
@@ -193,9 +194,10 @@
if (crawlStartURL != null && crawlStartURL.length() > 0) {
int maxPages = Integer.parseInt((String)parameters.get("crawlmaxpages"));
int maxDepth = Integer.parseInt((String)parameters.get("crawldepth"));
+ String crawlScopeURL = (String)parameters.get("scope");
String realmID = parameters.get("realmid").toString();
- importSite(crawlStartURL, maxPages, maxDepth, realmID);
+ importSite(crawlStartURL, crawlScopeURL, maxPages, maxDepth, realmID);
EventLog eventLog = (EventLog)session.getAttribute(SESSION_ATTR_EVENT_LOG);
if (eventLog != null) {
@@ -278,24 +280,36 @@
/**
* Crawls and external site and imports it into a realm.
* @param crawlStartURL
+ * @param crawlScopeURL comma-separated list of scope urls
* @param maxPages
* @param maxDepth
* @param realmID
* @throws Exception
*/
- protected void importSite(String crawlStartURL, int maxPages, int maxDepth, String realmID) throws Exception {
- String crawlScopeURL = crawlStartURL;
- URL url = new URL(crawlStartURL);
- String path = url.getPath();
- if (path.length() != 0 && !path.endsWith("/") && path.indexOf("/") > -1) {
- crawlScopeURL = crawlStartURL.substring(0, crawlStartURL.lastIndexOf("/"));
+ protected void importSite(String crawlStartURL, String crawlScopeURL, int maxPages, int maxDepth, String realmID) throws Exception {
+ String[] crawlScopeURLs = null;
+ if (crawlScopeURL == null || crawlScopeURL.length() == 0) {
+ String path = new URL(crawlStartURL).getPath();
+ crawlScopeURLs = new String[1];
+ if (path.length() != 0 && !path.endsWith("/") && path.indexOf("/") > -1) {
+ crawlScopeURLs[0] = crawlStartURL.substring(0, crawlStartURL.lastIndexOf("/"));
+ } else {
+ crawlScopeURLs[0] = crawlStartURL;
+ }
+ } else {
+ crawlScopeURLs = crawlScopeURL.split(",");
}
String dumpDir = System.getProperty("java.io.tmpdir") + File.separator + "import_" + System.currentTimeMillis();
- DumpingCrawler crawler = new DumpingCrawler(crawlStartURL, crawlScopeURL, dumpDir);
+ DumpingCrawler crawler = new DumpingCrawler(crawlStartURL, crawlScopeURLs, dumpDir);
crawler.setMaxPages(maxPages);
crawler.setMaxDepth(maxDepth);
+ DownloadParameters downloadParams = new DownloadParameters();
+ downloadParams = downloadParams.changeMaxPageSize(-1);
+ crawler.setDownloadParameters(downloadParams);
+
+
EventLog eventLog = new EventLog();
crawler.addLinkListener(eventLog);
crawler.addCrawlListener(eventLog);
@@ -389,6 +403,12 @@
crawlMaxPagesFieldElement.setAttributeNS(NAMESPACE, "samplevalue", "100");
crawlMaxPagesFieldElement.appendChild(document.createTextNode("crawlmaxpages"));
+ Element scopeFieldElement = (Element) inputFieldsElement.appendChild(document.createElementNS(NAMESPACE, "input"));
+ scopeFieldElement.setAttributeNS(NAMESPACE, "name", "scope");
+ scopeFieldElement.setAttributeNS(NAMESPACE, "required", "false");
+ scopeFieldElement.setAttributeNS(NAMESPACE, "samplevalue", "http://www.foo.bar");
+ scopeFieldElement.appendChild(document.createTextNode("scope"));
+
return document;
}
}
Modified: public/yanel/trunk/src/resources/add-realm/xslt/add-realm.xsl
===================================================================
--- public/yanel/trunk/src/resources/add-realm/xslt/add-realm.xsl 2007-03-22 21:41:20 UTC (rev 23375)
+++ public/yanel/trunk/src/resources/add-realm/xslt/add-realm.xsl 2007-03-22 22:27:51 UTC (rev 23376)
@@ -15,6 +15,7 @@
<xsl:param name="realmid" select="''" />
<xsl:param name="realmname" select="''" />
<xsl:param name="url" select="''" />
+ <xsl:param name="scope" select="''" />
<xsl:param name="fslocation" select="''" />
<xsl:param name="crawldepth" select="''" />
<xsl:param name="crawlmaxpages" select="''" />
@@ -22,6 +23,7 @@
<xsl:param name="realmid-prop-exists" select="''" />
<xsl:param name="realmname-prop-exists" select="''" />
<xsl:param name="url-prop-exists" select="''" />
+ <xsl:param name="scope-prop-exists" select="''" />
<xsl:param name="fslocation-prop-exists" select="''" />
<xsl:param name="crawldepth-prop-exists" select="''" />
<xsl:param name="crawlmaxpages-prop-exists" select="''" />
@@ -112,6 +114,9 @@
<xsl:when test="position()='6' and $crawlmaxpages-prop-exists = 'true'">
<xsl:call-template name="style-display-none"/>
</xsl:when>
+ <xsl:when test="position()='7' and $scope-prop-exists = 'true'">
+ <xsl:call-template name="style-display-none"/>
+ </xsl:when>
<xsl:otherwise/>
</xsl:choose>
@@ -144,6 +149,9 @@
<xsl:when test="position()='6' and not(contains($crawlmaxpages, 'ERROR:'))">
<xsl:attribute name="value"><xsl:value-of select="$crawlmaxpages" /></xsl:attribute>
</xsl:when>
+ <xsl:when test="position()='7' and not(contains($scope, 'ERROR:'))">
+ <xsl:attribute name="value"><xsl:value-of select="$scope" /></xsl:attribute>
+ </xsl:when>
<xsl:otherwise>
<xsl:attribute name="value"></xsl:attribute>
</xsl:otherwise>
@@ -220,6 +228,17 @@
</xsl:otherwise>
</xsl:choose>
</xsl:when>
+ <xsl:when test="position()='7'">
+ <xsl:attribute name="value"><xsl:value-of select="$scope" /></xsl:attribute>
+ <xsl:choose>
+ <xsl:when test="$scope-prop-exists != 'false'">
+ <xsl:attribute name="type">hidden</xsl:attribute>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:attribute name="type">text</xsl:attribute>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:when>
<xsl:otherwise>
<xsl:attribute name="value"></xsl:attribute>
</xsl:otherwise>
@@ -296,6 +315,17 @@
</font>
</td>
</xsl:when>
+ <xsl:when test="position()='7' and contains($scope, 'ERROR:')">
+ <td>
+  <font color="red">
+ <i18n:message>
+ <xsl:attribute name="key">
+ <xsl:value-of select="substring-after($scope, 'ERROR:')"/>
+ </xsl:attribute>
+ </i18n:message>
+ </font>
+ </td>
+ </xsl:when>
<xsl:otherwise>
<td>
 
More information about the Yanel-commits
mailing list