[zeppelin] branch master updated: [ZEPPELIN-5903] Improve LRU NoteCache - Add a cleanup (#4590)

[pdallig]

This is an automated email from the ASF dual-hosted git repository.

pdallig pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new d5eee9f160 [ZEPPELIN-5903] Improve LRU NoteCache - Add a cleanup 
(#4590)
d5eee9f160 is described below

commit d5eee9f160fbb2030574f2cfcdf651a3918abb79
Author: Philipp Dallig <philipp.dal...@gmail.com>
AuthorDate: Tue May 2 08:39:06 2023 +0200

    [ZEPPELIN-5903] Improve LRU NoteCache - Add a cleanup (#4590)
    
    * Improve LRU NoteCache - Add a cleanup
    
    * Remove debug code
    
    Co-authored-by: Guanhua Li <guanhua...@foxmail.com>
    
    ---------
    
    Co-authored-by: Guanhua Li <guanhua...@foxmail.com>
---
 .../org/apache/zeppelin/notebook/NoteManager.java  | 26 +++++++++++++++++++---
 .../apache/zeppelin/notebook/NoteManagerTest.java  | 12 ++++++++--
 2 files changed, 33 insertions(+), 5 deletions(-)

diff --git 
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java 
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
index e2d393af54..69c83030d6 100644
--- 
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
+++ 
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/notebook/NoteManager.java
@@ -21,7 +21,7 @@ package org.apache.zeppelin.notebook;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashMap;
+import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
@@ -736,7 +736,6 @@ public class NoteManager {
         if (size() <= NoteCache.this.threshold) {
           return false;
         }
-
         final Note eldestNote = eldest.getValue();
         final Lock lock = eldestNote.getLock().writeLock();
         if (lock.tryLock()) { // avoid eviction in case the note is in use
@@ -748,11 +747,32 @@ public class NoteManager {
         } else {
           LOGGER.info("Can not evict note {}, because the write lock can not 
be acquired. {} notes currently loaded.",
               eldestNote.getId(), size());
+          cleanupCache();
           return false;
         }
       }
-    }
 
+      private void cleanupCache() {
+        Iterator<Map.Entry<String, Note>> iterator = 
this.entrySet().iterator();
+        int count = 0;
+        // if size >= shrinked_size and have next() try remove
+        while ((this.size() - 1) >= NoteCache.this.threshold && 
iterator.hasNext()) {
+          Map.Entry<String, Note> noteEntry = iterator.next();
+          final Note note = noteEntry.getValue();
+          final Lock lock = note.getLock().writeLock();
+          if (lock.tryLock()) { // avoid eviction in case the note is in use
+            try {
+              iterator.remove(); // remove LRU element from LinkedHashMap
+              LOGGER.debug("Remove note {} from LRU Cache", note.getId());
+              ++count;
+            } finally {
+              lock.unlock();
+            }
+          }
+        }
+        LOGGER.info("The cache cleanup removes {} entries", count);
+      }
+    }
   }
 
 
diff --git 
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
 
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
index f8d58dd272..f825a97b37 100644
--- 
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
+++ 
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/notebook/NoteManagerTest.java
@@ -45,7 +45,6 @@ public class NoteManagerTest {
   @Rule
   public ExpectedException thrown = ExpectedException.none();
 
-
   @Before
   public void setUp() throws IOException {
     conf = ZeppelinConfiguration.create();
@@ -151,8 +150,9 @@ public class NoteManagerTest {
     }
     assertEquals(cacheThreshold, noteManager.getCacheSize());
 
-    // add cache + 1
+    // add cache + 1 with read flag
     Note noteNew2 = createNote("/prod/notenew2");
+    noteNew2.getLock().readLock().lock();
     noteManager.addNote(noteNew2, AuthenticationInfo.ANONYMOUS);
 
     // since all notes in the cache are with a read lock, the cache grows
@@ -162,6 +162,14 @@ public class NoteManagerTest {
     noteManager.removeNote(noteNew2.getId(), AuthenticationInfo.ANONYMOUS);
     assertFalse(noteManager.containsNote(noteNew2.getPath()));
     assertEquals(cacheThreshold, noteManager.getCacheSize());
+
+    // add cache + 1 without read flag
+    Note noteNew3 = createNote("/prod/notenew3");
+    noteManager.addNote(noteNew3, AuthenticationInfo.ANONYMOUS);
+
+    // since all dirty notes in the cache are with a read flag, the cache 
removes noteNew3, because it has no read flag
+    assertEquals(cacheThreshold, noteManager.getCacheSize());
+    assertTrue(noteManager.containsNote(noteNew3.getPath()));
   }
 
   @Test