Lucene timeout/NativeFSLock at publishing

Steps to reproduce

Two mall mediapackages finished processing about the same time. Log:

2011-03-24 22:36:05 ERROR (SolrException:139) - org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.lucene.store.Lock.obtain(Lock.java:85)
at org.apache.lucene.index.IndexWriter.init(IndexWriter.java:1565)
at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1421)
at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:191)
at org.apache.solr.update.UpdateHandler.createMainIndexWriter(UpdateHandler.java:98)
at org.apache.solr.update.DirectUpdateHandler2.openWriter(DirectUpdateHandler2.java:173)
at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:220)
at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:61)
at org.apache.solr.handler.XMLLoader.processUpdate(XMLLoader.java:139)
at org.apache.solr.handler.XMLLoader.load(XMLLoader.java:69)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:54)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:131)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1316)
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:139)
at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:105)
at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:64)
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:234)
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:255)
at org.opencastproject.workflow.handler.PublishWorkflowOperationHandler.start(PublishWorkflowOperationHandler.java:167)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.start(WorkflowOperationWorker.java:216)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.execute(WorkflowOperationWorker.java:138)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.runWorkflowOperation(WorkflowServiceImpl.java:646)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.process(WorkflowServiceImpl.java:1345)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1594)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1572)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)

2011-03-24 22:36:05 ERROR (WorkflowOperationWorker:156) - Workflow operation 'publish' failed
org.opencastproject.search.api.SearchException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:261)
at org.opencastproject.workflow.handler.PublishWorkflowOperationHandler.start(PublishWorkflowOperationHandler.java:167)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.start(WorkflowOperationWorker.java:216)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.execute(WorkflowOperationWorker.java:138)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.runWorkflowOperation(WorkflowServiceImpl.java:646)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.process(WorkflowServiceImpl.java:1345)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1594)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1572)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:240)
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:255)
... 12 more
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:153)
at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:105)
at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:64)
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:234)
... 13 more
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:141)
... 16 more
Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.lucene.store.Lock.obtain(Lock.java:85)
at org.apache.lucene.index.IndexWriter.init(IndexWriter.java:1565)
at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1421)
at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:191)
at org.apache.solr.update.UpdateHandler.createMainIndexWriter(UpdateHandler.java:98)
at org.apache.solr.update.DirectUpdateHandler2.openWriter(DirectUpdateHandler2.java:173)
at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:220)
at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:61)
at org.apache.solr.handler.XMLLoader.processUpdate(XMLLoader.java:139)
at org.apache.solr.handler.XMLLoader.load(XMLLoader.java:69)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:54)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:131)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1316)
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:139)
... 16 more

2011-03-24 22:36:15 ERROR (SolrIndexWriter:252) - SolrIndexWriter was not closed prior to finalize(), indicates a bug – POSSIBLE RESOURCE LEAK!!!

/data/matterhorn/shared/workspace/http_media.uct.ac.za/static/Unscheduled-HealthSciences-NewLearningCentre-1300892241414/catalog-1/episode.xml
2011-03-24 22:36:43 ERROR (SolrException:139) - org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.lucene.store.Lock.obtain(Lock.java:85)
at org.apache.lucene.index.IndexWriter.init(IndexWriter.java:1565)
at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1421)
at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:191)
at org.apache.solr.update.UpdateHandler.createMainIndexWriter(UpdateHandler.java:98)
at org.apache.solr.update.DirectUpdateHandler2.openWriter(DirectUpdateHandler2.java:173)
at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:220)
at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:61)
at org.apache.solr.handler.XMLLoader.processUpdate(XMLLoader.java:139)
at org.apache.solr.handler.XMLLoader.load(XMLLoader.java:69)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:54)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:131)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1316)
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:139)
at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:105)
at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:64)
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:234)
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:255)
at org.opencastproject.workflow.handler.PublishWorkflowOperationHandler.start(PublishWorkflowOperationHandler.java:167)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.start(WorkflowOperationWorker.java:216)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.execute(WorkflowOperationWorker.java:138)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.runWorkflowOperation(WorkflowServiceImpl.java:646)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.process(WorkflowServiceImpl.java:1345)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1594)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1572)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)

2011-03-24 22:36:43 ERROR (WorkflowOperationWorker:156) - Workflow operation 'publish' failed
org.opencastproject.search.api.SearchException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:261)
at org.opencastproject.workflow.handler.PublishWorkflowOperationHandler.start(PublishWorkflowOperationHandler.java:167)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.start(WorkflowOperationWorker.java:216)
at org.opencastproject.workflow.impl.WorkflowOperationWorker.execute(WorkflowOperationWorker.java:138)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.runWorkflowOperation(WorkflowServiceImpl.java:646)
at org.opencastproject.workflow.impl.WorkflowServiceImpl.process(WorkflowServiceImpl.java:1345)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1594)
at org.opencastproject.workflow.impl.WorkflowServiceImpl$JobRunner.call(WorkflowServiceImpl.java:1572)
at java.util.concurrent.FutureTask$Sync.innerRun(FutureTask.java:303)
at java.util.concurrent.FutureTask.run(FutureTask.java:138)
at java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:886)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:908)
at java.lang.Thread.run(Thread.java:662)
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:240)
at org.opencastproject.search.impl.SearchServiceImpl.add(SearchServiceImpl.java:255)
... 12 more
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:153)
at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:105)
at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:64)
at org.opencastproject.search.impl.solr.SolrIndexManager.add(SolrIndexManager.java:234)
... 13 more
Caused by: org.apache.solr.client.solrj.SolrServerException: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:141)
... 16 more
Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/data/matterhorn/search/data/index/lucene-e955f8c7e9c23a955dd48fb858ea8413-write.lock
at org.apache.lucene.store.Lock.obtain(Lock.java:85)
at org.apache.lucene.index.IndexWriter.init(IndexWriter.java:1565)
at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:1421)
at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:191)
at org.apache.solr.update.UpdateHandler.createMainIndexWriter(UpdateHandler.java:98)
at org.apache.solr.update.DirectUpdateHandler2.openWriter(DirectUpdateHandler2.java:173)
at org.apache.solr.update.DirectUpdateHandler2.addDoc(DirectUpdateHandler2.java:220)
at org.apache.solr.update.processor.RunUpdateProcessor.processAdd(RunUpdateProcessorFactory.java:61)
at org.apache.solr.handler.XMLLoader.processUpdate(XMLLoader.java:139)
at org.apache.solr.handler.XMLLoader.load(XMLLoader.java:69)
at org.apache.solr.handler.ContentStreamHandlerBase.handleRequestBody(ContentStreamHandlerBase.java:54)
at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:131)
at org.apache.solr.core.SolrCore.execute(SolrCore.java:1316)
at org.apache.solr.client.solrj.embedded.EmbeddedSolrServer.request(EmbeddedSolrServer.java:139)
... 16 more

Status

Assignee

Lovemore Nalube

Reporter

Lovemore Nalube

Severity

Non Functioning

Tags (folksonomy)

None

Fix versions

Affects versions

Priority

Blocker
Configure